├── .ci └── test-project │ ├── README.md │ ├── packages │ ├── magictestlib.tar.gz │ ├── magictestlib │ │ └── package.py │ ├── magictestlib_cached │ │ └── package.py │ └── uberenv-magictestlib │ │ ├── package.py │ │ └── uberenv-magictestlib.tar.gz │ ├── repo.yaml │ ├── spack_configs │ ├── darwin │ │ └── spack.yaml │ ├── linux_ubuntu_22 │ │ └── spack.yaml │ └── toss_4_x86_64_ib │ │ └── spack.yaml │ └── uberenv_configs │ ├── dev-build.json │ ├── install.json │ └── uberenv-pkg.json ├── .github └── workflows │ ├── linux.yml │ └── macos.yml ├── .gitignore ├── .gitlab-ci.yml ├── .readthedocs.yml ├── LICENSE ├── Makefile ├── README.md ├── RELEASE-NOTES.md ├── deprecated_uberenv.py ├── docs ├── requirements.txt └── sphinx │ ├── conf.py │ └── index.rst ├── gen_spack_env_script.py └── uberenv.py /.ci/test-project/README.md: -------------------------------------------------------------------------------- 1 | # test-project 2 | Tests build options in linux and macos. Each build option (uberenv-pkg, install, dev-build) is tested in linux, and currently only uberenv-pkg is tested in macos (see `.github/workflows`). The `--package-final-phase` option is also tested for dev-build and install, since it is a commonly used option in projects that utilize Uberenv. 3 | 4 | # Testing locally 5 | If you wish to test locally, look at `.github/workflows`, and follow a job's "Run Uberenv" commands (e.g. "build_uberenv_mode"). 6 | -------------------------------------------------------------------------------- /.ci/test-project/packages/magictestlib.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/uberenv/a76a2942f0b575f7d9047dfb87d699e2b6d18f12/.ci/test-project/packages/magictestlib.tar.gz -------------------------------------------------------------------------------- /.ci/test-project/packages/magictestlib/package.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 3 | # 4 | # Produced at the Lawrence Livermore National Laboratory 5 | # 6 | # LLNL-CODE-666778 7 | # 8 | # All rights reserved. 9 | # 10 | # This file is part of Conduit. 11 | # 12 | # For details, see https://lc.llnl.gov/conduit/. 13 | # 14 | # Please also read conduit/LICENSE 15 | # 16 | # Redistribution and use in source and binary forms, with or without 17 | # modification, are permitted provided that the following conditions are met: 18 | # 19 | # * Redistributions of source code must retain the above copyright notice, 20 | # this list of conditions and the disclaimer below. 21 | # 22 | # * Redistributions in binary form must reproduce the above copyright notice, 23 | # this list of conditions and the disclaimer (as noted below) in the 24 | # documentation and/or other materials provided with the distribution. 25 | # 26 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 27 | # be used to endorse or promote products derived from this software without 28 | # specific prior written permission. 29 | # 30 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 31 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 32 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 34 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 35 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 40 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 41 | # POSSIBILITY OF SUCH DAMAGE. 42 | # 43 | ############################################################################### 44 | 45 | import glob 46 | import os 47 | import shutil 48 | import socket 49 | from os import environ as env 50 | 51 | import llnl.util.tty as tty 52 | 53 | from spack.package import * 54 | 55 | 56 | def cmake_cache_entry(name, value, vtype=None): 57 | """ 58 | Helper that creates CMake cache entry strings used in 59 | 'host-config' files. 60 | """ 61 | if vtype is None: 62 | if value == "ON" or value == "OFF": 63 | vtype = "BOOL" 64 | else: 65 | vtype = "PATH" 66 | return 'set({0} "{1}" CACHE {2} "")\n\n'.format(name, value, vtype) 67 | 68 | 69 | class Magictestlib(Package): 70 | """Magictestlib""" 71 | 72 | homepage = "http://example.com/" 73 | url = "http://example.com/" 74 | git = "http://example.com/" 75 | 76 | version('1.0.0', 'c8b277080a00041cfc4f64619e31f6d6',preferred=True) 77 | 78 | depends_on('zlib') 79 | 80 | depends_on("c", type="build") 81 | depends_on("cxx", type="build") 82 | 83 | ################################### 84 | # build phases used by this package 85 | ################################### 86 | phases = ['hostconfig'] 87 | 88 | def url_for_version(self, version): 89 | dummy_tar_path = os.path.abspath(os.path.join(__file__, "../../magictestlib.tar.gz")) 90 | url = "file://" + dummy_tar_path 91 | return url 92 | 93 | 94 | def _get_host_config_path(self, spec): 95 | sys_type = spec.architecture 96 | # if on llnl systems, we can use the SYS_TYPE 97 | if "SYS_TYPE" in env: 98 | sys_type = env["SYS_TYPE"] 99 | 100 | compiler_str = f"{self['c'].name}-{self['c'].version}" 101 | host_config_path = ( 102 | f"{socket.gethostname()}-{sys_type}-{compiler_str}-magictestlib-{spec.dag_hash()}.cmake" 103 | ) 104 | dest_dir = self.stage.source_path 105 | host_config_path = os.path.abspath(join_path(dest_dir, host_config_path)) 106 | return host_config_path 107 | 108 | def hostconfig(self, spec, prefix): 109 | """ 110 | This method creates a mock 'host-config' file. 111 | """ 112 | if not os.path.isdir(spec.prefix): 113 | os.mkdir(spec.prefix) 114 | 115 | ####################### 116 | # Compiler Info 117 | ####################### 118 | c_compiler = env["CC"] 119 | cpp_compiler = env["CXX"] 120 | 121 | ####################################################################### 122 | # Directly fetch the names of the actual compilers to create a 123 | # 'host config' file that works outside of the spack install env. 124 | ####################################################################### 125 | # get hostconfig name 126 | host_cfg_fname = self._get_host_config_path(spec) 127 | 128 | cfg = open(host_cfg_fname, "w") 129 | cfg.write("##################################\n") 130 | cfg.write("# spack generated host-config\n") 131 | cfg.write("##################################\n") 132 | ####################### 133 | # Spack Compiler info 134 | ####################### 135 | cfg.write("#######\n") 136 | cfg.write("# using %s compiler spec\n" % spec.compiler) 137 | cfg.write("#######\n\n") 138 | cfg.write("# c compiler used by spack\n") 139 | cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) 140 | cfg.write("# cpp compiler used by spack\n") 141 | cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) 142 | 143 | ###### 144 | # ZLIB 145 | cfg.write(cmake_cache_entry("ZLIB_DIR", spec['zlib'].prefix)) 146 | 147 | ####################### 148 | # Finish host-config 149 | ####################### 150 | cfg.write("##################################\n") 151 | cfg.write("# end spack generated host-config\n") 152 | cfg.write("##################################\n") 153 | cfg.close() 154 | 155 | host_cfg_fname = os.path.abspath(host_cfg_fname) 156 | tty.info("spack generated host-config file: " + host_cfg_fname) 157 | 158 | # Copy the generated host-config to install directory for downstream use 159 | @run_before("install") 160 | def copy_host_config(self): 161 | src = self._get_host_config_path(self.spec) 162 | dst = join_path(self.spec.prefix, os.path.basename(src)) 163 | copy(src, dst) 164 | -------------------------------------------------------------------------------- /.ci/test-project/packages/magictestlib_cached/package.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 3 | # 4 | # Produced at the Lawrence Livermore National Laboratory 5 | # 6 | # LLNL-CODE-666778 7 | # 8 | # All rights reserved. 9 | # 10 | # This file is part of Conduit. 11 | # 12 | # For details, see https://lc.llnl.gov/conduit/. 13 | # 14 | # Please also read conduit/LICENSE 15 | # 16 | # Redistribution and use in source and binary forms, with or without 17 | # modification, are permitted provided that the following conditions are met: 18 | # 19 | # * Redistributions of source code must retain the above copyright notice, 20 | # this list of conditions and the disclaimer below. 21 | # 22 | # * Redistributions in binary form must reproduce the above copyright notice, 23 | # this list of conditions and the disclaimer (as noted below) in the 24 | # documentation and/or other materials provided with the distribution. 25 | # 26 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 27 | # be used to endorse or promote products derived from this software without 28 | # specific prior written permission. 29 | # 30 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 31 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 32 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 34 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 35 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 40 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 41 | # POSSIBILITY OF SUCH DAMAGE. 42 | # 43 | ############################################################################### 44 | 45 | from spack.package import * 46 | 47 | import os 48 | import socket 49 | from os import environ as env 50 | 51 | import llnl.util.tty as tty 52 | 53 | 54 | class MagictestlibCached(CachedCMakePackage): 55 | """MagictestlibCached""" 56 | 57 | homepage = "http://example.com/" 58 | url = "http://example.com/" 59 | git = "http://example.com/" 60 | 61 | version('1.0.0', 'c8b277080a00041cfc4f64619e31f6d6',preferred=True) 62 | 63 | depends_on('hdf5~mpi') 64 | 65 | depends_on("c", type="build") 66 | depends_on("cxx", type="build") 67 | 68 | def url_for_version(self, version): 69 | dummy_tar_path = os.path.abspath(os.path.join(__file__, "../../magictestlib.tar.gz")) 70 | url = "file://" + dummy_tar_path 71 | return url 72 | 73 | def _get_sys_type(self, spec): 74 | sys_type = spec.architecture 75 | # if on llnl systems, we can use the SYS_TYPE 76 | if "SYS_TYPE" in env: 77 | sys_type = env["SYS_TYPE"] 78 | return sys_type 79 | 80 | @property 81 | def cache_name(self): 82 | hostname = socket.gethostname() 83 | if "SYS_TYPE" in env: 84 | # Are we on a LLNL system then strip node number 85 | hostname = hostname.rstrip('1234567890') 86 | return "{0}-{1}-{2}@{3}.cmake".format( 87 | hostname, 88 | self._get_sys_type(self.spec), 89 | self.spec.compiler.name, 90 | self.spec.compiler.version 91 | ) 92 | 93 | def initconfig_package_entries(self): 94 | spec = self.spec 95 | entries = [] 96 | 97 | # TPL locations 98 | entries.append("#------------------{0}".format("-" * 60)) 99 | entries.append("# TPLs") 100 | entries.append("#------------------{0}\n".format("-" * 60)) 101 | 102 | entries.append(cmake_cache_path('HDF5_DIR', spec["hdf5"].prefix)) 103 | 104 | return entries 105 | -------------------------------------------------------------------------------- /.ci/test-project/packages/uberenv-magictestlib/package.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 3 | # 4 | # Produced at the Lawrence Livermore National Laboratory 5 | # 6 | # LLNL-CODE-666778 7 | # 8 | # All rights reserved. 9 | # 10 | # This file is part of Conduit. 11 | # 12 | # For details, see https://lc.llnl.gov/conduit/. 13 | # 14 | # Please also read conduit/LICENSE 15 | # 16 | # Redistribution and use in source and binary forms, with or without 17 | # modification, are permitted provided that the following conditions are met: 18 | # 19 | # * Redistributions of source code must retain the above copyright notice, 20 | # this list of conditions and the disclaimer below. 21 | # 22 | # * Redistributions in binary form must reproduce the above copyright notice, 23 | # this list of conditions and the disclaimer (as noted below) in the 24 | # documentation and/or other materials provided with the distribution. 25 | # 26 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 27 | # be used to endorse or promote products derived from this software without 28 | # specific prior written permission. 29 | # 30 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 31 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 32 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 34 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 35 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 40 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 41 | # POSSIBILITY OF SUCH DAMAGE. 42 | # 43 | ############################################################################### 44 | 45 | from spack.package import * 46 | 47 | import socket 48 | import os 49 | 50 | from .magictestlib import Magictestlib 51 | 52 | class UberenvMagictestlib(Magictestlib): 53 | """Uberenv Test""" 54 | version('1.0.0', 'c8b277080a00041cfc4f64619e31f6d6',preferred=True) 55 | 56 | ################################### 57 | # build phases used by this package 58 | ################################### 59 | phases = ['hostconfig'] 60 | 61 | def url_for_version(self, version): 62 | dummy_tar_path = os.path.abspath(os.path.join(__file__, "../uberenv-magictestlib.tar.gz")) 63 | url = "file://" + dummy_tar_path 64 | return url 65 | 66 | def hostconfig(self, spec, prefix): 67 | super().hostconfig(spec, prefix) 68 | src = self._get_host_config_path(self.spec) 69 | dst = join_path(self.spec.prefix, os.path.basename(src)) 70 | copy(src, dst) 71 | -------------------------------------------------------------------------------- /.ci/test-project/packages/uberenv-magictestlib/uberenv-magictestlib.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/uberenv/a76a2942f0b575f7d9047dfb87d699e2b6d18f12/.ci/test-project/packages/uberenv-magictestlib/uberenv-magictestlib.tar.gz -------------------------------------------------------------------------------- /.ci/test-project/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: magictestlib 3 | -------------------------------------------------------------------------------- /.ci/test-project/spack_configs/darwin/spack.yaml: -------------------------------------------------------------------------------- 1 | # Spack env file for mac 2 | # Allow Spack to find and use compilers 3 | spack: 4 | packages: 5 | cmake: 6 | externals: 7 | - spec: cmake@3.14.5 8 | prefix: /usr 9 | buildable: false 10 | pkg-config: 11 | externals: 12 | - spec: pkg-config@0.27.1 13 | prefix: /usr 14 | buildable: false 15 | zlib: 16 | externals: 17 | - spec: zlib@1.2.7 18 | prefix: /usr 19 | buildable: false 20 | -------------------------------------------------------------------------------- /.ci/test-project/spack_configs/linux_ubuntu_22/spack.yaml: -------------------------------------------------------------------------------- 1 | # Spack env file for ubuntu 22 2 | spack: 3 | packages: 4 | cmake: 5 | externals: 6 | - spec: cmake@3.25.2 7 | prefix: /usr 8 | buildable: false 9 | hdf5: 10 | require: "@1.14.0" 11 | pkg-config: 12 | externals: 13 | - spec: pkg-config@0.27.1 14 | prefix: /usr 15 | buildable: false 16 | zlib: 17 | externals: 18 | - spec: zlib@1.2.7 19 | prefix: /usr 20 | buildable: false 21 | -------------------------------------------------------------------------------- /.ci/test-project/spack_configs/toss_4_x86_64_ib/spack.yaml: -------------------------------------------------------------------------------- 1 | # Spack env file for toss4 2 | spack: 3 | packages: 4 | cmake: 5 | externals: 6 | - spec: cmake@3.19.2 7 | prefix: /usr/tce 8 | buildable: false 9 | pkg-config: 10 | externals: 11 | - spec: pkg-config@1.4.2 12 | prefix: /usr 13 | buildable: false 14 | zlib: 15 | externals: 16 | - spec: zlib@1.2.11 17 | prefix: /usr 18 | buildable: false 19 | compilers:: 20 | - compiler: 21 | environment: {} 22 | extra_rpaths: [] 23 | flags: {} 24 | modules: 25 | - gcc/10.3.1 26 | operating_system: rhel8 27 | paths: 28 | cc: /usr/tce/packages/gcc/gcc-10.3.1/bin/gcc 29 | cxx: /usr/tce/packages/gcc/gcc-10.3.1/bin/g++ 30 | f77: /usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran 31 | fc: /usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran 32 | spec: gcc@10.3.1 33 | target: x86_64 34 | -------------------------------------------------------------------------------- /.ci/test-project/uberenv_configs/dev-build.json: -------------------------------------------------------------------------------- 1 | { 2 | "package_name" : "magictestlib_cached", 3 | "package_version" : "1.0.0", 4 | "package_source_dir" : ".", 5 | "spack_build_mode" : "dev-build", 6 | "spack_configs_path": "spack_configs", 7 | "spack_packages_path": "packages", 8 | "spack_url": "https://github.com/spack/spack", 9 | "spack_branch": "develop" 10 | } 11 | -------------------------------------------------------------------------------- /.ci/test-project/uberenv_configs/install.json: -------------------------------------------------------------------------------- 1 | { 2 | "package_name" : "magictestlib_cached", 3 | "package_version" : "1.0.0", 4 | "package_source_dir" : ".", 5 | "spack_build_mode" : "install", 6 | "spack_configs_path": "spack_configs", 7 | "spack_packages_path": "packages", 8 | "spack_url": "https://github.com/spack/spack", 9 | "spack_externals": "cmake pkg-config zlib", 10 | "spack_branch": "develop" 11 | } 12 | -------------------------------------------------------------------------------- /.ci/test-project/uberenv_configs/uberenv-pkg.json: -------------------------------------------------------------------------------- 1 | { 2 | "package_name" : "magictestlib", 3 | "package_version" : "1.0.0", 4 | "package_source_dir" : ".", 5 | "spack_build_mode" : "uberenv-pkg", 6 | "spack_configs_path": "spack_configs", 7 | "spack_packages_path": "packages", 8 | "spack_url": "https://github.com/spack/spack", 9 | "spack_branch": "develop" 10 | } 11 | -------------------------------------------------------------------------------- /.github/workflows/linux.yml: -------------------------------------------------------------------------------- 1 | # Test all Spack build modes in linux 2 | 3 | name: linux 4 | 5 | on: 6 | pull_request: 7 | branches: [ main ] 8 | 9 | env: 10 | BASE_PACKAGES: binutils gcc g++ gfortran cmake python3 perl git git-lfs curl wget tar unzip build-essential 11 | 12 | jobs: 13 | # Tests uberenv-pkg Spack build mode 14 | build_uberenv_mode: 15 | name: Uberenv Build Mode (Linux) 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Install Deps 19 | run: | 20 | sudo apt-get update 21 | sudo apt-get install $BASE_PACKAGES 22 | - uses: actions/checkout@v3 23 | - name: Run Uberenv 24 | run: | 25 | cd .ci/test-project 26 | python3 ../../uberenv.py --project-json=uberenv_configs/uberenv-pkg.json --spack-env-file=spack_configs/linux_ubuntu_22/spack.yaml 27 | cat uberenv_libs/*.cmake 28 | # Tests dev-build Spack build mode 29 | build_dev_build_full_mode: 30 | name: Dev Build Full Mode (Linux) 31 | runs-on: ubuntu-latest 32 | steps: 33 | - name: Install Deps 34 | run: | 35 | sudo apt-get update 36 | sudo apt-get install $BASE_PACKAGES 37 | - uses: actions/checkout@v3 38 | - name: Run Uberenv 39 | run: | 40 | cp .ci/test-project/packages/magictestlib.tar.gz . && tar -xvf magictestlib.tar.gz && cp src/* . 41 | cd .ci/test-project 42 | python3 ../../uberenv.py --project-json=uberenv_configs/dev-build.json --spack-env-file=spack_configs/linux_ubuntu_22/spack.yaml 43 | find . -type f -name uberenv_conduit_hello -exec {} \; 44 | # Tests install Spack build mode 45 | build_install_full_mode: 46 | name: Install Full Mode (Linux) 47 | runs-on: ubuntu-latest 48 | steps: 49 | - name: Install Deps 50 | run: | 51 | sudo apt-get update 52 | sudo apt-get install $BASE_PACKAGES 53 | - uses: actions/checkout@v3 54 | - name: Run Uberenv 55 | run: | 56 | cd .ci/test-project 57 | python3 ../../uberenv.py --project-json=uberenv_configs/install.json --spack-env-file=spack_configs/linux_ubuntu_22/spack.yaml 58 | ./uberenv_libs/magictestlib_cached-install/bin/uberenv_conduit_hello 59 | # Tests dev-build Spack build mode stopping at initconfig phase 60 | build_dev_build_initconfig_mode: 61 | name: Dev Build Initconfig Mode (Linux) 62 | runs-on: ubuntu-latest 63 | steps: 64 | - name: Install Deps 65 | run: | 66 | sudo apt-get update 67 | sudo apt-get install $BASE_PACKAGES 68 | - uses: actions/checkout@v3 69 | - name: Run Uberenv 70 | run: | 71 | cp .ci/test-project/packages/magictestlib.tar.gz . && tar -xvf magictestlib.tar.gz && cp src/* . 72 | cd .ci/test-project 73 | python3 ../../uberenv.py --project-json=uberenv_configs/dev-build.json --spack-env-file=spack_configs/linux_ubuntu_22/spack.yaml --package-final-phase=initconfig 74 | # Tests install Spack build mode stopping at initconfig phase 75 | build_install_initconfig_mode: 76 | name: Install Initconfig Mode (Linux) 77 | runs-on: ubuntu-latest 78 | steps: 79 | - name: Install Deps 80 | run: | 81 | sudo apt-get update 82 | sudo apt-get install $BASE_PACKAGES 83 | - uses: actions/checkout@v3 84 | - name: Run Uberenv 85 | run: | 86 | cd .ci/test-project 87 | python3 ../../uberenv.py --project-json=uberenv_configs/install.json --spack-env-file=spack_configs/linux_ubuntu_22/spack.yaml --package-final-phase=initconfig 88 | # Tests install Spack build mode with setup-only and skip-setup options 89 | build_setup_options_mode: 90 | name: Testing Setup-only and Skip-setup (Linux) 91 | runs-on: ubuntu-latest 92 | steps: 93 | - name: Install Deps 94 | run: | 95 | sudo apt-get update 96 | sudo apt-get install $BASE_PACKAGES 97 | - uses: actions/checkout@v3 98 | - name: Run Uberenv 99 | run: | 100 | cd .ci/test-project 101 | python3 ../../uberenv.py --project-json=uberenv_configs/install.json --spack-env-file=spack_configs/linux_ubuntu_22/spack.yaml --setup-only 102 | python3 ../../uberenv.py --project-json=uberenv_configs/install.json --spack-env-file=spack_configs/linux_ubuntu_22/spack.yaml --skip-setup 103 | ./uberenv_libs/magictestlib_cached-install/bin/uberenv_conduit_hello 104 | # Tests install Spack build mode and generating a spack.yaml 105 | build_spack_yaml: 106 | name: Generate spack.yaml (Linux) 107 | runs-on: ubuntu-latest 108 | steps: 109 | - name: Install Deps 110 | run: | 111 | sudo apt-get update 112 | sudo apt-get install $BASE_PACKAGES 113 | - uses: actions/checkout@v3 114 | - name: Run Uberenv 115 | run: | 116 | cd .ci/test-project 117 | python3 ../../uberenv.py --project-json=uberenv_configs/install.json --spec="%gcc ^hdf5@1.14.0" 118 | ./uberenv_libs/magictestlib_cached-install/bin/uberenv_conduit_hello 119 | cat ./uberenv_libs/spack.yaml 120 | -------------------------------------------------------------------------------- /.github/workflows/macos.yml: -------------------------------------------------------------------------------- 1 | name: macos 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | 7 | jobs: 8 | # Test uberenv-pkg Spack build mode in macos 9 | build_uberenv_mode: 10 | name: Uberenv Build Mode (MacOS) 11 | runs-on: macos-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Run Uberenv 15 | run: | 16 | cd .ci/test-project 17 | python3 ../../uberenv.py --project-json=uberenv_configs/uberenv-pkg.json 18 | cat uberenv_libs/*.cmake 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | s_env.sh 4 | _docs_html 5 | .ci/test-project/uberenv_libs 6 | spack-build-* 7 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 3 | # 4 | # Produced at the Lawrence Livermore National Laboratory 5 | # 6 | # LLNL-CODE-666778 7 | # 8 | # All rights reserved. 9 | # 10 | # This file is part of Conduit. 11 | # 12 | # For details, see https://lc.llnl.gov/conduit/. 13 | # 14 | # Please also read conduit/LICENSE 15 | # 16 | # Redistribution and use in source and binary forms, with or without 17 | # modification, are permitted provided that the following conditions are met: 18 | # 19 | # * Redistributions of source code must retain the above copyright notice, 20 | # this list of conditions and the disclaimer below. 21 | # 22 | # * Redistributions in binary form must reproduce the above copyright notice, 23 | # this list of conditions and the disclaimer (as noted below) in the 24 | # documentation and/or other materials provided with the distribution. 25 | # 26 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 27 | # be used to endorse or promote products derived from this software without 28 | # specific prior written permission. 29 | # 30 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 31 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 32 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 34 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 35 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 40 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 41 | # POSSIBILITY OF SUCH DAMAGE. 42 | # 43 | ############################################################################### 44 | 45 | # Triggers Conduit pipeline asking for uberenv update 46 | # UPDATE_UBERENV variable, when set, will tell conduit to use the value as 47 | # a reference to the uberenv env revision to checkout on github. 48 | 49 | # Note: the following pipelines have been disabled because 50 | # - they need to be fixed 51 | # - they cause quota issues (conduit CI is quite space consuming) 52 | # - they may cause too much trafic in projects pipeline views 53 | 54 | #trigger-conduit: 55 | # variables: 56 | # UPDATE_UBERENV: $CI_COMMIT_REF_NAME 57 | # trigger: 58 | # project: radiuss/conduit 59 | # branch: feature/update-uberenv 60 | # strategy: depend 61 | 62 | trigger-serac: 63 | variables: 64 | UPDATE_UBERENV: $CI_COMMIT_REF_NAME 65 | ALLOC_BANK: radiuss 66 | trigger: 67 | project: smith/serac 68 | branch: develop 69 | strategy: depend 70 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/sphinx/conf.py 17 | 18 | # Optionally build your docs in additional formats such as PDF and ePub 19 | formats: all 20 | 21 | # Set requirements required to build your docs 22 | python: 23 | install: 24 | - requirements: docs/requirements.txt 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 2 | 3 | Produced at the Lawrence Livermore National Laboratory 4 | 5 | LLNL-CODE-666778 6 | 7 | All rights reserved. 8 | 9 | This file is part of Conduit. 10 | 11 | For details, see: http://software.llnl.gov/conduit/. 12 | 13 | Please also read conduit/LICENSE 14 | 15 | Redistribution and use in source and binary forms, with or without 16 | modification, are permitted provided that the following conditions are met: 17 | 18 | * Redistributions of source code must retain the above copyright notice, 19 | this list of conditions and the disclaimer below. 20 | 21 | * Redistributions in binary form must reproduce the above copyright notice, 22 | this list of conditions and the disclaimer (as noted below) in the 23 | documentation and/or other materials provided with the distribution. 24 | 25 | * Neither the name of the LLNS/LLNL nor the names of its contributors may 26 | be used to endorse or promote products derived from this software without 27 | specific prior written permission. 28 | 29 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 30 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 | ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 33 | LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 34 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 35 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 36 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 37 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 38 | STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 39 | IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 | POSSIBILITY OF SUCH DAMAGE. 41 | 42 | Additional BSD Notice 43 | 44 | 1. This notice is required to be provided under our contract with the U.S. 45 | Department of Energy (DOE). This work was produced at Lawrence 46 | Livermore National Laboratory under Contract No. DE-AC52-07NA27344 with 47 | the DOE. 48 | 49 | 2. Neither the United States Government nor Lawrence Livermore National 50 | Security, LLC nor any of their employees, makes any warranty, express 51 | or implied, or assumes any liability or responsibility for the 52 | accuracy, completeness, or usefulness of any information, apparatus, 53 | product, or process disclosed, or represents that its use would not 54 | infringe privately-owned rights. 55 | 56 | 3. Also, reference herein to any specific commercial products, process, 57 | or services by trade name, trademark, manufacturer or otherwise does 58 | not necessarily constitute or imply its endorsement, recommendation, 59 | or favoring by the United States Government or Lawrence Livermore 60 | National Security, LLC. The views and opinions of authors expressed 61 | herein do not necessarily state or reflect those of the United 62 | States Government or Lawrence Livermore National Security, LLC, and 63 | shall not be used for advertising or product endorsement purposes. 64 | 65 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | default: 3 | sphinx-build -E -a -b html docs/sphinx/ _docs_html 4 | 5 | clean: 6 | rm -rf _docs_html -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # uberenv 2 | Automates using a package manager to build and deploy software. 3 | 4 | [![Read the Docs](https://readthedocs.org/projects/uberenv/badge/?version=latest)](https://uberenv.readthedocs.io) 5 | 6 | Uberenv is a python script that helps automate building 7 | third-party dependencies for development and deployment. 8 | 9 | Uberenv uses Spack (https://www.spack.io/) on Unix-based systems (e.g. Linux and macOS) 10 | and Vcpkg (https://github.com/microsoft/vcpkg) on Windows systems. 11 | 12 | Uberenv was released as part of the Conduit project (https://github.com/LLNL/conduit/). 13 | It is included in-source in several projects, this repo is used to hold the latest reference version. 14 | 15 | For more details, see Uberenv's documention: 16 | 17 | https://uberenv.readthedocs.io 18 | 19 | You can also find details about how it is used in Conduit's documentation: 20 | 21 | https://llnl-conduit.readthedocs.io/en/latest/building.html#building-conduit-and-third-party-dependencies 22 | 23 | Conduit's source repo also serves as an example for uberenv and spack configuration files, etc: 24 | 25 | https://github.com/LLNL/conduit/tree/master/scripts/uberenv 26 | -------------------------------------------------------------------------------- /RELEASE-NOTES.md: -------------------------------------------------------------------------------- 1 | # Uberenv Software Release Notes 2 | 3 | Notes describing significant changes in each Uberenv release are documented 4 | in this file. 5 | 6 | The format of this file is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 7 | 8 | The Uberenv project release numbers follow [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 9 | 10 | ## Unreleased 11 | 12 | ### Added 13 | - Adds `--skip-setup` option, which skips installing and setting up Spack and installs only 14 | (using pre-setup Spack). Useful for air-gapped builds. 15 | - Allow `--package_final_phase` command to be used for install build option, which controls 16 | after which phase Spack should stop. 17 | - Allow projects to force specifying `--prefix` on command line via new project option: 18 | `force_commandline_prefix`. This is useful when the default `uberenv_libs` can not exist 19 | inside of your project's source repository. 20 | - Adds support for Windows builds using [Vcpkg]. 21 | - Adds the `--triplet` command line argument for setting the Vcpkg build configuration. 22 | - Adds the `--vcpkg-ports-path` command line argument for setting the path to the vcpkg ports directory. 23 | - Adds support for the spack `--reuse` option for more relaxed tpl/dependency matching. 24 | - Adds support for `--upstream` with `--setup-only`. 25 | - Allow to prevent Uberenv from installing clingo setting `spack_setup_clingo` 26 | to `false` in `.uberenv_config.json`. 27 | - Adds the `--spack-debug` option to run spack spec/install commands in debug mode. 28 | - Adds the `--spack-allow-deprecated` option, to allow spack to build packages marked deprecated. 29 | 30 | ### Changed 31 | - All spack specs are now expressed inside single quotes to protect the parsing of complex flags. 32 | - Added ability to have multiple packages directories that will get copied into spack on top of 33 | each other via project configuration option: `spack_packages_path` 34 | - Pretty print various options to screen for readability 35 | - Allow `.uberenv_config.json` to live at the same level as `uberenv.py` 36 | - No longer removes symlinks when using the directory of `uberenv.py` 37 | - Reduce Spack's git history to a bare minimum 38 | - Uberenv now requires python version 3.3 or above. 39 | - Rather than using pip, Uberenv uses `spack bootstrap now` to install clingo. 40 | - Removes Spack concretizer options, since clingo is the only option in newer Spack. (You can still disable clingo install.) 41 | 42 | ### Fixed 43 | 44 | 45 | [Vcpkg]: https://github.com/microsoft/vcpkg 46 | -------------------------------------------------------------------------------- /deprecated_uberenv.py: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | "exec" "python3" "-u" "-B" "$0" "$@" 3 | ############################################################################### 4 | # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 5 | # 6 | # Produced at the Lawrence Livermore National Laboratory 7 | # 8 | # LLNL-CODE-666778 9 | # 10 | # All rights reserved. 11 | # 12 | # This file is part of Conduit. 13 | # 14 | # For details, see https://lc.llnl.gov/conduit/. 15 | # 16 | # Please also read conduit/LICENSE 17 | # 18 | # Redistribution and use in source and binary forms, with or without 19 | # modification, are permitted provided that the following conditions are met: 20 | # 21 | # * Redistributions of source code must retain the above copyright notice, 22 | # this list of conditions and the disclaimer below. 23 | # 24 | # * Redistributions in binary form must reproduce the above copyright notice, 25 | # this list of conditions and the disclaimer (as noted below) in the 26 | # documentation and/or other materials provided with the distribution. 27 | # 28 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 29 | # be used to endorse or promote products derived from this software without 30 | # specific prior written permission. 31 | # 32 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 33 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 34 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 35 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 36 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 37 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 41 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 42 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 43 | # POSSIBILITY OF SUCH DAMAGE. 44 | # 45 | ############################################################################### 46 | 47 | """ 48 | file: deprecated_uberenv.py 49 | 50 | description: automates using a package manager to install a project. 51 | Uses spack on Unix-based systems and Vcpkg on Windows-based systems. 52 | 53 | note: This file is deprecated. It uses spack activate and spack deactivate, 54 | which are no longer supported, having been deprecated in Spack v0.18 and 55 | completely removed in v0.19. This file is meant allow use of the original 56 | Uberenv to allow time to switch to the new Uberenv featuring Spack 57 | Environments. 58 | 59 | """ 60 | 61 | import os 62 | import sys 63 | import subprocess 64 | import shutil 65 | import socket 66 | import platform 67 | import json 68 | import datetime 69 | import glob 70 | import re 71 | 72 | from optparse import OptionParser 73 | from distutils.version import LooseVersion 74 | from functools import partial 75 | 76 | from os import environ as env 77 | from os.path import join as pjoin 78 | from os.path import abspath as pabs 79 | 80 | # Since we use subprocesses, flushing prints allows us to keep logs in 81 | # order. 82 | print = partial(print, flush=True) 83 | 84 | def sexe(cmd,ret_output=False,echo=False): 85 | """ Helper for executing shell commands. """ 86 | if echo: 87 | print("[exe: {0}]".format(cmd)) 88 | if ret_output: 89 | p = subprocess.Popen(cmd, 90 | shell=True, 91 | stdout=subprocess.PIPE, 92 | stderr=subprocess.STDOUT) 93 | out = p.communicate()[0] 94 | out = out.decode('utf8') 95 | return p.returncode,out 96 | else: 97 | return subprocess.call(cmd,shell=True) 98 | 99 | 100 | def parse_args(): 101 | "Parses args from command line" 102 | parser = OptionParser() 103 | parser.add_option("--install", 104 | action="store_true", 105 | dest="install", 106 | default=False, 107 | help="Install `package_name`, not just its dependencies.") 108 | 109 | # where to install 110 | parser.add_option("--prefix", 111 | dest="prefix", 112 | default=None, 113 | help="destination directory") 114 | 115 | # Spack spec without preceding package name 116 | parser.add_option("--spec", 117 | dest="spec", 118 | default=None, 119 | help="Spack spec without preceding package name") 120 | 121 | # for vcpkg, what architecture to target 122 | parser.add_option("--triplet", 123 | dest="vcpkg_triplet", 124 | default=None, 125 | help="vcpkg architecture triplet") 126 | 127 | # optional location of spack mirror 128 | parser.add_option("--mirror", 129 | dest="mirror", 130 | default=None, 131 | help="spack mirror directory") 132 | 133 | # flag to create mirror 134 | parser.add_option("--create-mirror", 135 | action="store_true", 136 | dest="create_mirror", 137 | default=False, 138 | help="Create spack mirror") 139 | 140 | # optional location of spack upstream 141 | parser.add_option("--upstream", 142 | dest="upstream", 143 | default=None, 144 | help="add an external spack instance as upstream") 145 | 146 | # optional spack --reuse concretizer behaviour 147 | parser.add_option("--reuse", 148 | dest="reuse", 149 | default=False, 150 | help="Use spack v0.17+ --reuse functionality for spec, install and dev-build.") 151 | 152 | # this option allows a user to explicitly to select a 153 | # group of spack settings files (compilers.yaml , packages.yaml) 154 | parser.add_option("--spack-config-dir", 155 | dest="spack_config_dir", 156 | default=None, 157 | help="dir with spack settings files (compilers.yaml, packages.yaml, etc)") 158 | 159 | # this option allows a user to set the directory for their vcpkg ports on Windows 160 | parser.add_option("--vcpkg-ports-path", 161 | dest="vcpkg_ports_path", 162 | default=None, 163 | help="dir with vckpkg ports") 164 | 165 | # overrides package_name 166 | parser.add_option("--package-name", 167 | dest="package_name", 168 | default=None, 169 | help="override the default package name") 170 | 171 | # uberenv spack tpl build mode 172 | parser.add_option("--spack-build-mode", 173 | dest="spack_build_mode", 174 | default=None, 175 | help="set mode used to build third party dependencies with spack" 176 | "(options: 'dev-build' 'uberenv-pkg' 'install' " 177 | "[default: 'dev-build'] )\n") 178 | 179 | # spack debug mode 180 | parser.add_option("--spack-debug", 181 | dest="spack_debug", 182 | action="store_true", 183 | default=False, 184 | help="add debug option to spack spec/install commands") 185 | 186 | # spack allow deprecated packages 187 | parser.add_option("--spack-allow-deprecated", 188 | dest="spack_allow_deprecated", 189 | action="store_true", 190 | default=False, 191 | help="add --deprecated to spack install commands") 192 | 193 | # controls after which package phase spack should stop 194 | parser.add_option("--package-final-phase", 195 | dest="package_final_phase", 196 | default=None, 197 | help="override the default phase after which spack should stop") 198 | 199 | # controls source_dir spack should use to build the package 200 | parser.add_option("--package-source-dir", 201 | dest="package_source_dir", 202 | default=None, 203 | help="override the default source dir spack should use") 204 | 205 | # a file that holds settings for a specific project 206 | # using uberenv.py 207 | parser.add_option("--project-json", 208 | dest="project_json", 209 | default=pjoin(uberenv_script_dir(),"project.json"), 210 | help="uberenv project settings json file") 211 | 212 | # option to explicitly set the number of build jobs 213 | parser.add_option("-j", 214 | dest="build_jobs", 215 | default=None, 216 | help="Explicitly set build jobs") 217 | 218 | # flag to use insecure curl + git 219 | parser.add_option("-k", 220 | action="store_true", 221 | dest="ignore_ssl_errors", 222 | default=False, 223 | help="Ignore SSL Errors") 224 | 225 | # option to force a pull of the package manager 226 | parser.add_option("--pull", 227 | action="store_true", 228 | dest="repo_pull", 229 | default=False, 230 | help="Pull from package manager, if repo already exists") 231 | 232 | # option to force for clean of packages specified to 233 | # be cleaned in the project.json 234 | parser.add_option("--clean", 235 | action="store_true", 236 | dest="spack_clean", 237 | default=False, 238 | help="Force uninstall of packages specified in project.json") 239 | 240 | # option to tell spack to run tests 241 | parser.add_option("--run_tests", 242 | action="store_true", 243 | dest="run_tests", 244 | default=False, 245 | help="Invoke build tests during spack install") 246 | 247 | # option to init osx sdk env flags 248 | parser.add_option("--macos-sdk-env-setup", 249 | action="store_true", 250 | dest="macos_sdk_env_setup", 251 | default=False, 252 | help="Set several env vars to select OSX SDK settings." 253 | "This was necessary for older versions of macOS " 254 | " but can cause issues with macOS versions >= 10.13. " 255 | " so it is disabled by default.") 256 | 257 | # option to stop after spack download and setup 258 | parser.add_option("--setup-only", 259 | action="store_true", 260 | dest="setup_only", 261 | default=False, 262 | help="Only download and setup Spack. No further Spack command will be run.") 263 | 264 | # option to skip spack download and setup 265 | parser.add_option("--skip-setup", 266 | action="store_true", 267 | dest="skip_setup", 268 | default=False, 269 | help="Only install (using pre-setup Spack).") 270 | 271 | 272 | ############### 273 | # parse args 274 | ############### 275 | opts, extras = parser.parse_args() 276 | # we want a dict b/c the values could 277 | # be passed without using optparse 278 | opts = vars(opts) 279 | if opts["spack_config_dir"] is not None: 280 | opts["spack_config_dir"] = pabs(opts["spack_config_dir"]) 281 | if not os.path.isdir(opts["spack_config_dir"]): 282 | print("[ERROR: invalid spack config dir: {0} ]".format(opts["spack_config_dir"])) 283 | sys.exit(-1) 284 | # if rel path is given for the mirror, we need to evaluate here -- before any 285 | # chdirs to avoid confusion related to what it is relative to. 286 | # (it should be relative to where uberenv is run from, so it matches what you expect 287 | # from shell completion, etc) 288 | if not is_windows() and opts["mirror"] is not None: 289 | if not opts["mirror"].startswith("http") and not os.path.isabs(opts["mirror"]): 290 | opts["mirror"] = pabs(opts["mirror"]) 291 | return opts, extras 292 | 293 | 294 | def pretty_print_dictionary(dictionary): 295 | for key, value in dictionary.items(): 296 | print(" {0}: {1}".format(key, value)) 297 | 298 | def uberenv_script_dir(): 299 | # returns the directory of the uberenv.py script 300 | return os.path.dirname(os.path.abspath(__file__)) 301 | 302 | def load_json_file(json_file): 303 | # reads json file 304 | return json.load(open(json_file)) 305 | 306 | def is_darwin(): 307 | return "darwin" in platform.system().lower() 308 | 309 | def is_windows(): 310 | return "windows" in platform.system().lower() 311 | 312 | def find_project_config(opts): 313 | project_json_file = opts["project_json"] 314 | # Default case: "project.json" seats next to uberenv.py or is given on command line. 315 | if os.path.isfile(project_json_file): 316 | return project_json_file 317 | # Submodule case: Look for ".uberenv_config.json" in current then search parent dirs 318 | else: 319 | lookup_path = pabs(uberenv_script_dir()) 320 | end_of_search = False 321 | while not end_of_search: 322 | if os.path.dirname(lookup_path) == lookup_path: 323 | end_of_search = True 324 | project_json_file = pjoin(lookup_path,".uberenv_config.json") 325 | if os.path.isfile(project_json_file): 326 | return project_json_file 327 | else: 328 | lookup_path = pabs(os.path.join(lookup_path, os.pardir)) 329 | print("ERROR: No configuration json file found") 330 | sys.exit(-1) 331 | 332 | 333 | class UberEnv(): 334 | """ Base class for package manager """ 335 | 336 | def __init__(self, opts, extra_opts): 337 | self.opts = opts 338 | self.extra_opts = extra_opts 339 | 340 | # load project settings 341 | self.project_opts = load_json_file(opts["project_json"]) 342 | 343 | # setup main package name 344 | self.pkg_name = self.set_from_args_or_json("package_name") 345 | 346 | # Set project.json defaults 347 | if not "force_commandline_prefix" in self.project_opts: 348 | self.project_opts["force_commandline_prefix"] = False 349 | 350 | print("[uberenv project settings: ") 351 | pretty_print_dictionary(self.project_opts) 352 | print("]") 353 | 354 | print("[uberenv command line options: ") 355 | pretty_print_dictionary(self.opts) 356 | print("]") 357 | 358 | ########################### 359 | # basic spack helpers 360 | ########################### 361 | def spack_exe_path(self): 362 | return pjoin(self.dest_dir, "spack/bin/spack") 363 | 364 | def spack_version(self): 365 | res, out = sexe('{0} --version'.format(self.spack_exe_path()), ret_output=True) 366 | return LooseVersion(out[:-1]) 367 | 368 | def setup_paths_and_dirs(self): 369 | self.uberenv_path = uberenv_script_dir() 370 | 371 | # setup destination paths 372 | if not self.opts["prefix"]: 373 | if self.project_opts["force_commandline_prefix"]: 374 | # project has specified prefix must be on command line 375 | print("[ERROR: --prefix flag for library destination is required]") 376 | sys.exit(1) 377 | # otherwise set default 378 | self.opts["prefix"] = "uberenv_libs" 379 | 380 | self.dest_dir = pabs(self.opts["prefix"]) 381 | 382 | # print a warning if the dest path already exists 383 | if not os.path.isdir(self.dest_dir): 384 | os.mkdir(self.dest_dir) 385 | else: 386 | print("[info: destination '{0}' already exists]".format(self.dest_dir)) 387 | 388 | def set_from_args_or_json(self,setting, optional=True): 389 | """ 390 | When optional=False: 391 | If the setting key is not in the json file, error and raise an exception. 392 | When optional=True: 393 | If the setting key is not in the json file or opts, return None. 394 | """ 395 | setting_value = None 396 | try: 397 | setting_value = self.project_opts[setting] 398 | except (KeyError): 399 | if not optional: 400 | print("ERROR: '{0}' must at least be defined in project.json".format(setting)) 401 | raise 402 | if self.opts[setting]: 403 | setting_value = self.opts[setting] 404 | return setting_value 405 | 406 | def set_from_json(self,setting, optional=True): 407 | """ 408 | When optional=False: 409 | If the setting key is not in the json file, error and raise an exception. 410 | When optional=True: 411 | If the setting key is not in the json file or opts, return None. 412 | """ 413 | setting_value = None 414 | try: 415 | setting_value = self.project_opts[setting] 416 | except (KeyError): 417 | if not optional: 418 | print("ERROR: '{0}' must at least be defined in project.json".format(setting)) 419 | raise 420 | return setting_value 421 | 422 | def detect_platform(self): 423 | # find supported sets of compilers.yaml, packages,yaml 424 | res = None 425 | if is_darwin(): 426 | res = "darwin" 427 | elif "SYS_TYPE" in os.environ.keys(): 428 | sys_type = os.environ["SYS_TYPE"].lower() 429 | res = sys_type 430 | return res 431 | 432 | 433 | class VcpkgEnv(UberEnv): 434 | """ Helper to clone vcpkg and install libraries on Windows """ 435 | 436 | def __init__(self, opts, extra_opts): 437 | UberEnv.__init__(self,opts,extra_opts) 438 | 439 | # setup architecture triplet 440 | self.vcpkg_triplet = self.set_from_args_or_json("vcpkg_triplet") 441 | print("Vcpkg triplet: {}".format(self.vcpkg_triplet)) 442 | if self.vcpkg_triplet is None: 443 | self.vcpkg_triplet = os.getenv("VCPKG_DEFAULT_TRIPLET", "x86-windows") 444 | 445 | def setup_paths_and_dirs(self): 446 | # get the current working path, and the glob used to identify the 447 | # package files we want to hot-copy to vcpkg 448 | 449 | UberEnv.setup_paths_and_dirs(self) 450 | 451 | # Find path to vcpkg ports 452 | _errmsg = "" 453 | if self.opts["vcpkg_ports_path"]: 454 | # Command line option case 455 | self.vcpkg_ports_path = pabs(self.opts["vcpkg_ports_path"]) 456 | _errmsg = "Given path for command line option `vcpkg-ports-path` does not exist" 457 | elif "vcpkg_ports_path" in self.project_opts: 458 | # .uberenv_config.json case 459 | new_path = self.project_opts["vcpkg_ports_path"] 460 | if new_path is not None: 461 | self.vcpkg_ports_path = pabs(new_path) 462 | _errmsg = "Given path in config file option 'vcpkg_ports_path' does not exist" 463 | else: 464 | # next to uberenv.py script (backwards compatibility) 465 | self.vcpkg_ports_path = pabs(pjoin(self.uberenv_path, "vcpkg_ports")) 466 | _errmsg = "Could not find any directory for vcpkg ports. " \ 467 | "Use either command line option 'vcpkg-ports-path', " \ 468 | "config file option 'vcpkg_ports_path', or " \ 469 | "defaulted directory 'vcpkg_ports' next to 'uberenv.py'" 470 | 471 | if not os.path.isdir(self.vcpkg_ports_path): 472 | print("[ERROR: {0}: {1}]".format(_errmsg, self.vcpkg_ports_path)) 473 | sys.exit(1) 474 | 475 | # setup path for vcpkg repo 476 | print("[installing to: {0}]".format(self.dest_dir)) 477 | self.dest_vcpkg = pjoin(self.dest_dir,"vcpkg") 478 | 479 | if os.path.isdir(self.dest_vcpkg): 480 | print("[info: destination '{0}' already exists]".format(self.dest_vcpkg)) 481 | 482 | def clone_repo(self): 483 | if not os.path.isdir(self.dest_vcpkg): 484 | # compose clone command for the dest path, vcpkg url and branch 485 | vcpkg_branch = self.project_opts.get("vcpkg_branch", "master") 486 | vcpkg_url = self.project_opts.get("vcpkg_url", "https://github.com/microsoft/vcpkg") 487 | 488 | print("[info: cloning vcpkg '{0}' branch from {1} into {2}]" 489 | .format(vcpkg_branch,vcpkg_url, self.dest_vcpkg)) 490 | 491 | os.chdir(self.dest_dir) 492 | 493 | clone_opts = ("-c http.sslVerify=false " 494 | if self.opts["ignore_ssl_errors"] else "") 495 | 496 | clone_cmd = "git {0} clone --single-branch -b {1} {2} vcpkg".format(clone_opts, vcpkg_branch,vcpkg_url) 497 | sexe(clone_cmd, echo=True) 498 | 499 | # optionally, check out a specific commit 500 | if "vcpkg_commit" in self.project_opts: 501 | sha1 = self.project_opts["vcpkg_commit"] 502 | print("[info: using vcpkg commit {0}]".format(sha1)) 503 | os.chdir(self.dest_vcpkg) 504 | sexe("git checkout {0}".format(sha1),echo=True) 505 | 506 | if self.opts["repo_pull"]: 507 | # do a pull to make sure we have the latest 508 | os.chdir(self.dest_vcpkg) 509 | sexe("git stash", echo=True) 510 | res = sexe("git pull", echo=True) 511 | if res != 0: 512 | #Usually untracked files that would be overwritten 513 | print("[ERROR: Git failed to pull]") 514 | sys.exit(-1) 515 | 516 | 517 | # Bootstrap vcpkg 518 | os.chdir(self.dest_vcpkg) 519 | print("[info: bootstrapping vcpkg]") 520 | sexe("bootstrap-vcpkg.bat -disableMetrics") 521 | 522 | def patch(self): 523 | """ hot-copy our ports into vcpkg """ 524 | 525 | import distutils.dir_util 526 | 527 | dest_vcpkg_ports = pjoin(self.dest_vcpkg, "ports") 528 | 529 | print("[info: copying from {0} to {1}]".format(self.vcpkg_ports_path, dest_vcpkg_ports)) 530 | distutils.dir_util.copy_tree(self.vcpkg_ports_path, dest_vcpkg_ports) 531 | 532 | 533 | def clean_build(self): 534 | pass 535 | 536 | def show_info(self): 537 | os.chdir(self.dest_vcpkg) 538 | print("[info: Details for package '{0}']".format(self.pkg_name)) 539 | sexe("vcpkg.exe search " + self.pkg_name, echo=True) 540 | 541 | print("[info: Dependencies for package '{0}']".format(self.pkg_name)) 542 | sexe("vcpkg.exe depend-info " + self.pkg_name, echo=True) 543 | 544 | def create_mirror(self): 545 | pass 546 | 547 | def use_mirror(self): 548 | pass 549 | 550 | def install(self): 551 | 552 | os.chdir(self.dest_vcpkg) 553 | install_cmd = "vcpkg.exe " 554 | install_cmd += "install {0}:{1}".format(self.pkg_name, self.vcpkg_triplet) 555 | 556 | res = sexe(install_cmd, echo=True) 557 | 558 | # Running the install_cmd eventually generates the host config file, 559 | # which we copy to the target directory. 560 | src_hc = pjoin(self.dest_vcpkg, "installed", self.vcpkg_triplet, "include", self.pkg_name, "hc.cmake") 561 | hcfg_fname = pjoin(self.dest_dir, "{0}.{1}.cmake".format(platform.uname()[1], self.vcpkg_triplet)) 562 | print("[info: copying host config file to {0}]".format(hcfg_fname)) 563 | shutil.copy(os.path.abspath(src_hc), hcfg_fname) 564 | print("") 565 | print("[install complete!]") 566 | return res 567 | 568 | 569 | class SpackEnv(UberEnv): 570 | """ Helper to clone spack and install libraries on MacOS an Linux """ 571 | 572 | def __init__(self, opts, extra_opts): 573 | UberEnv.__init__(self,opts,extra_opts) 574 | self.pkg_version = self.set_from_json("package_version") 575 | self.pkg_src_dir = self.set_from_args_or_json("package_source_dir", True) 576 | self.pkg_final_phase = self.set_from_args_or_json("package_final_phase",True) 577 | # get build mode 578 | self.build_mode = self.set_from_args_or_json("spack_build_mode",True) 579 | # default spack build mode is dev-build 580 | if self.build_mode is None: 581 | self.build_mode = "dev-build" 582 | # NOTE: install always overrides the build mode to "install" 583 | if self.opts["install"]: 584 | self.build_mode = "install" 585 | # if we are using fake package mode, adjust the pkg name 586 | if self.build_mode == "uberenv-pkg": 587 | self.pkg_name = "uberenv-" + self.pkg_name 588 | 589 | print("[uberenv spack build mode: {0}]".format(self.build_mode)) 590 | self.packages_paths = [] 591 | self.spec_hash = "" 592 | self.use_install = False 593 | 594 | if "spack_concretizer" in self.project_opts and self.project_opts["spack_concretizer"] == "clingo": 595 | self.use_clingo = True 596 | if "spack_setup_clingo" in self.project_opts and self.project_opts["spack_setup_clingo"] == False: 597 | print("[info: clingo will not be installed by uberenv]") 598 | else: 599 | self.setup_clingo() 600 | else: 601 | self.use_clingo = False 602 | 603 | # Some additional setup for macos 604 | if is_darwin(): 605 | if opts["macos_sdk_env_setup"]: 606 | # setup osx deployment target and sdk settings 607 | setup_osx_sdk_env_vars() 608 | else: 609 | print("[skipping MACOSX env var setup]") 610 | 611 | # setup default spec 612 | if opts["spec"] is None: 613 | if is_darwin(): 614 | # Note: newer spack, for macOS we need to use `apple-clang` 615 | opts["spec"] = "%apple-clang" 616 | else: 617 | opts["spec"] = "%gcc" 618 | self.opts["spec"] = "@{0}{1}".format(self.pkg_version,opts["spec"]) 619 | elif not opts["spec"].startswith("@"): 620 | self.opts["spec"] = "@{0}{1}".format(self.pkg_version,opts["spec"]) 621 | else: 622 | self.opts["spec"] = "{0}".format(opts["spec"]) 623 | 624 | print("[spack spec: {0}]".format(self.opts["spec"])) 625 | 626 | # List of concretizer options not in all versions of spack 627 | # (to be checked if it exists after cloning spack) 628 | self.fresh_exists = False 629 | self.reuse_exists = False 630 | 631 | def check_concretizer_opts(self): 632 | cmd = "{0} help install".format(self.spack_exe_path()) 633 | print("[Checking for concretizer options...]") 634 | res, out = sexe( cmd, ret_output = True) 635 | if "--fresh" in out: 636 | self.fresh_exists = True 637 | print("[--fresh exists.]") 638 | if "--reuse" in out: 639 | self.reuse_exists = True 640 | print("[--reuse exists.]") 641 | 642 | def add_concretizer_opts(self, options): 643 | # reuse is now default in spack, if on and exists use that 644 | # otherwise use fresh if it exists 645 | if self.opts["reuse"] and self.reuse_exists: 646 | options += "--reuse " 647 | elif self.fresh_exists: 648 | options += "--fresh " 649 | return options 650 | 651 | def print_spack_python_info(self): 652 | cmd = "{0} python -c \"import sys; print(sys.executable);\"".format(self.spack_exe_path()) 653 | res, out = sexe( cmd, ret_output = True) 654 | print("[spack python: {0}]".format(out.strip())) 655 | 656 | def append_path_to_packages_paths(self, path, errorOnNonexistant=True): 657 | path = pabs(path) 658 | if not os.path.exists(path): 659 | if errorOnNonexistant: 660 | print("[ERROR: Given path in 'spack_packages_path' does not exist: {0}]".format(path)) 661 | sys.exit(1) 662 | else: 663 | return 664 | self.packages_paths.append(path) 665 | 666 | 667 | def setup_paths_and_dirs(self): 668 | # get the current working path, and the glob used to identify the 669 | # package files we want to hot-copy to spack 670 | 671 | UberEnv.setup_paths_and_dirs(self) 672 | 673 | # Find Spack yaml configs path (compilers.yaml, packages.yaml, etc.) 674 | 675 | # Next to uberenv.py (backwards compatility) 676 | spack_configs_path = pabs(pjoin(self.uberenv_path,"spack_configs")) 677 | 678 | # In project config file 679 | if "spack_configs_path" in self.project_opts.keys(): 680 | new_path = self.project_opts["spack_configs_path"] 681 | if new_path is not None: 682 | spack_configs_path = pabs(new_path) 683 | if not os.path.isdir(spack_configs_path): 684 | print("[ERROR: Given path in 'spack_configs_path' does not exist: {0}]".format(spack_configs_path)) 685 | sys.exit(1) 686 | 687 | # Test if the override option was used (--spack-config-dir) 688 | self.spack_config_dir = self.opts["spack_config_dir"] 689 | if self.spack_config_dir is None: 690 | # If command line option is not used, search for platform under 691 | # given directory 692 | uberenv_plat = self.detect_platform() 693 | if uberenv_plat is not None: 694 | self.spack_config_dir = pabs(pjoin(spack_configs_path,uberenv_plat)) 695 | 696 | # Find project level packages to override spack's internal packages 697 | if "spack_packages_path" in self.project_opts.keys(): 698 | # packages directories listed in project.json 699 | _paths = self.project_opts["spack_packages_path"] 700 | if not isinstance(_paths, list): 701 | # user gave a single string 702 | self.append_path_to_packages_paths(_paths) 703 | else: 704 | # user gave a list of strings 705 | for _path in _paths: 706 | self.append_path_to_packages_paths(_path) 707 | else: 708 | # default to packages living next to uberenv script if it exists 709 | self.append_path_to_packages_paths(pjoin(self.uberenv_path,"packages"), errorOnNonexistant=False) 710 | 711 | print("[installing to: {0}]".format(self.dest_dir)) 712 | 713 | self.dest_spack = pjoin(self.dest_dir,"spack") 714 | if os.path.isdir(self.dest_spack): 715 | print("[info: destination '{0}' already exists]".format(self.dest_spack)) 716 | 717 | if self.build_mode == "dev-build": 718 | self.pkg_src_dir = os.path.abspath(os.path.join(self.uberenv_path,self.pkg_src_dir)) 719 | if not os.path.isdir(self.pkg_src_dir): 720 | print("[ERROR: package_source_dir '{0}' does not exist]".format(self.pkg_src_dir)) 721 | sys.exit(-1) 722 | 723 | def find_spack_pkg_path_from_hash(self, pkg_name, pkg_hash): 724 | res, out = sexe("{0} find -p /{1}".format(self.spack_exe_path(), pkg_hash), ret_output = True) 725 | for l in out.split("\n"): 726 | # TODO: at least print a warning when several choices exist. This will 727 | # pick the first in the list. 728 | if l.startswith(pkg_name): 729 | return {"name": pkg_name, "path": l.split()[-1]} 730 | print("[ERROR: failed to find package from hash named '{0}' with hash '{1}']".format(pkg_name, pkg_hash)) 731 | sys.exit(-1) 732 | 733 | def find_spack_pkg_path(self, pkg_name, spec = ""): 734 | res, out = sexe("{0} find -p '{1}{2}'".format(self.spack_exe_path(),pkg_name,spec), ret_output = True) 735 | for l in out.split("\n"): 736 | # TODO: at least print a warning when several choices exist. This will 737 | # pick the first in the list. 738 | if l.startswith(pkg_name): 739 | return {"name": pkg_name, "path": l.split()[-1]} 740 | print("[ERROR: failed to find package from spec named '{0}' with spec '{1}']".format(pkg_name, spec)) 741 | sys.exit(-1) 742 | 743 | # Extract the first line of the full spec 744 | def read_spack_full_spec(self,pkg_name,spec): 745 | debug = "" 746 | if self.opts["spack_debug"]: 747 | debug = "--debug --stacktrace " 748 | 749 | res, out = sexe("{0} {1} spec '{2}{3}'".format(self.spack_exe_path(),debug,pkg_name,spec), ret_output=True) 750 | for l in out.split("\n"): 751 | if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0: 752 | return l.strip() 753 | 754 | def clone_repo(self): 755 | if not os.path.isdir(self.dest_spack): 756 | 757 | # compose clone command for the dest path, spack url and branch 758 | print("[info: cloning spack develop branch from github]") 759 | 760 | os.chdir(self.dest_dir) 761 | 762 | clone_opts = ("-c http.sslVerify=false " 763 | if self.opts["ignore_ssl_errors"] else "") 764 | 765 | spack_url = self.project_opts.get("spack_url", "https://github.com/spack/spack.git") 766 | spack_branch = self.project_opts.get("spack_branch", "develop") 767 | 768 | clone_cmd = "git {0} clone --single-branch --depth=1 -b {1} {2} spack".format(clone_opts, spack_branch, spack_url) 769 | sexe(clone_cmd, echo=True) 770 | 771 | if "spack_commit" in self.project_opts: 772 | # optionally, check out a specific commit 773 | os.chdir(pjoin(self.dest_dir,"spack")) 774 | sha1 = self.project_opts["spack_commit"] 775 | res, current_sha1 = sexe("git log -1 --pretty=%H", ret_output=True) 776 | if sha1 != current_sha1: 777 | print("[info: using spack commit {0}]".format(sha1)) 778 | sexe("git stash", echo=True) 779 | sexe("git fetch --depth=1 origin {0}".format(sha1),echo=True) 780 | res = sexe("git checkout {0}".format(sha1),echo=True) 781 | if res != 0: 782 | # Usually untracked files that would be overwritten 783 | print("[ERROR: Git failed to checkout]") 784 | sys.exit(-1) 785 | 786 | if self.opts["repo_pull"]: 787 | # do a pull to make sure we have the latest 788 | os.chdir(pjoin(self.dest_dir,"spack")) 789 | sexe("git stash", echo=True) 790 | res = sexe("git pull", echo=True) 791 | if res != 0: 792 | #Usually untracked files that would be overwritten 793 | print("[ERROR: Git failed to pull]") 794 | sys.exit(-1) 795 | 796 | def disable_spack_config_scopes(self,spack_dir): 797 | # disables all config scopes except "defaults", which we will 798 | # force our settings into 799 | spack_lib_config = pjoin(spack_dir,"lib","spack","spack","config.py") 800 | print("[disabling config scope (except defaults) in: {0}]".format(spack_lib_config)) 801 | cfg_script = open(spack_lib_config).read() 802 | # 803 | # For newer versions of spack, we can use the SPACK_DISABLE_LOCAL_CONFIG 804 | # env var plumbing. We patch it to True to make a permanent change. 805 | # 806 | # Note: This path does not disable the 'site' config, but disabling 'user' config 807 | # is our primary goal. 808 | # 809 | spack_disable_env_stmt = 'disable_local_config = "SPACK_DISABLE_LOCAL_CONFIG" in os.environ' 810 | spack_disable_env_stmt_perm = "disable_local_config = True" 811 | if cfg_script.count(spack_disable_env_stmt) > 0: 812 | cfg_script = cfg_script.replace(spack_disable_env_stmt, 813 | spack_disable_env_stmt_perm) 814 | # path for older versions of spack 815 | elif cfg_script.count(spack_disable_env_stmt_perm) == 0: 816 | for cfg_scope_stmt in ["('system', os.path.join(spack.paths.system_etc_path, 'spack')),", 817 | "('site', os.path.join(spack.paths.etc_path, 'spack')),", 818 | "('user', spack.paths.user_config_path)"]: 819 | cfg_script = cfg_script.replace(cfg_scope_stmt, 820 | "#DISABLED BY UBERENV: " + cfg_scope_stmt) 821 | open(spack_lib_config,"w").write(cfg_script) 822 | 823 | def patch(self): 824 | cfg_dir = self.spack_config_dir 825 | spack_dir = self.dest_spack 826 | 827 | # this is an opportunity to show spack python info post obtaining spack 828 | self.print_spack_python_info() 829 | 830 | # Check which concretizer this version of Spack has 831 | self.check_concretizer_opts() 832 | 833 | # force spack to use only "defaults" config scope 834 | self.disable_spack_config_scopes(spack_dir) 835 | spack_etc_defaults_dir = pjoin(spack_dir,"etc","spack","defaults") 836 | 837 | if cfg_dir is not None: 838 | # copy in "defaults" config.yaml 839 | config_yaml = pabs(pjoin(cfg_dir,"..","config.yaml")) 840 | if os.path.isfile(config_yaml): 841 | sexe("cp {0} {1}/".format(config_yaml, spack_etc_defaults_dir), echo=True) 842 | mirrors_yaml = pabs(pjoin(cfg_dir,"..","mirrors.yaml")) 843 | if os.path.isfile(mirrors_yaml): 844 | sexe("cp {0} {1}/".format(mirrors_yaml, spack_etc_defaults_dir), echo=True) 845 | 846 | # copy in other settings per platform 847 | print("[copying uberenv compiler and packages settings from {0}]".format(cfg_dir)) 848 | 849 | config_yaml = pjoin(cfg_dir,"config.yaml") 850 | mirrors_yaml = pjoin(cfg_dir,"mirrors.yaml") 851 | compilers_yaml = pjoin(cfg_dir,"compilers.yaml") 852 | packages_yaml = pjoin(cfg_dir,"packages.yaml") 853 | 854 | if os.path.isfile(config_yaml): 855 | sexe("cp {0} {1}/".format(config_yaml , spack_etc_defaults_dir ), echo=True) 856 | 857 | if os.path.isfile(mirrors_yaml): 858 | sexe("cp {0} {1}/".format(mirrors_yaml , spack_etc_defaults_dir ), echo=True) 859 | 860 | if os.path.isfile(compilers_yaml): 861 | sexe("cp {0} {1}/".format(compilers_yaml, spack_etc_defaults_dir ), echo=True) 862 | 863 | if os.path.isfile(packages_yaml): 864 | sexe("cp {0} {1}/".format(packages_yaml, spack_etc_defaults_dir ), echo=True) 865 | else: 866 | # let spack try to auto find compilers 867 | sexe("{0} compiler find".format(self.spack_exe_path()), echo=True) 868 | 869 | # hot-copy our packages into spack 870 | if len(self.packages_paths) > 0: 871 | dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages") 872 | for _base_path in self.packages_paths: 873 | _src_glob = pjoin(_base_path, "*") 874 | print("[copying patched packages from {0}]".format(_src_glob)) 875 | sexe("cp -Rf {0} {1}".format(_src_glob, dest_spack_pkgs)) 876 | 877 | # Update spack's config.yaml if clingo was requested 878 | if self.use_clingo: 879 | concretizer_cmd = "{0} config --scope defaults add config:concretizer:clingo".format(self.spack_exe_path()) 880 | res = sexe(concretizer_cmd, echo=True) 881 | if res != 0: 882 | print("[ERROR: Failed to update spack configuration to use new concretizer]") 883 | sys.exit(-1) 884 | 885 | def clean_build(self): 886 | # clean out any spack cached stuff (except build stages, downloads, & 887 | # spack's bootstrapping software) 888 | cln_cmd = "{0} clean --misc-cache --failures --python-cache".format(self.spack_exe_path()) 889 | res = sexe(cln_cmd, echo=True) 890 | 891 | # check if we need to force uninstall of selected packages 892 | if self.opts["spack_clean"]: 893 | if "spack_clean_packages" in self.project_opts: 894 | for cln_pkg in self.project_opts["spack_clean_packages"]: 895 | if self.find_spack_pkg_path(cln_pkg) is not None: 896 | unist_cmd = "{0} uninstall -f -y --all --dependents ".format(self.spack_exe_path()) + cln_pkg 897 | res = sexe(unist_cmd, echo=True) 898 | 899 | def show_info(self): 900 | # print version of spack 901 | print("[spack version: {0}]".format(self.spack_version())) 902 | 903 | # print concretized spec with install info 904 | # default case prints install status and 32 characters hash 905 | debug = "" 906 | if self.opts["spack_debug"]: 907 | debug = "--debug --stacktrace " 908 | 909 | options = "" 910 | options = self.add_concretizer_opts(options) 911 | options += "--install-status --very-long" 912 | spec_cmd = "{0} {1}spec {2} '{3}{4}'".format(self.spack_exe_path(),debug,options,self.pkg_name,self.opts["spec"]) 913 | 914 | res, out = sexe(spec_cmd, ret_output=True, echo=True) 915 | print(out) 916 | 917 | #Check if spec is already installed 918 | for line in out.split("\n"): 919 | # Example of matching line: ("status" "hash" "package"...) 920 | # [+] hf3cubkgl74ryc3qwen73kl4yfh2ijgd serac@develop%clang@10.0.0-apple~debug~devtools~glvis arch=darwin-mojave-x86_64 921 | if re.match(r"^(\[\+\]| - ) [a-z0-9]{32} " + re.escape(self.pkg_name), line): 922 | self.spec_hash = line.split(" ")[1].lstrip() 923 | # if spec already installed 924 | if line.startswith("[+]"): 925 | pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name,self.spec_hash) 926 | install_path = pkg_path["path"] 927 | # testing that the path exists is mandatory until Spack team fixes 928 | # https://github.com/spack/spack/issues/16329 929 | if os.path.isdir(install_path): 930 | print("[Warning: {0} {1} has already been installed in {2}]".format(self.pkg_name, self.opts["spec"],install_path)) 931 | print("[Warning: Uberenv will proceed using this directory]".format(self.pkg_name)) 932 | self.use_install = True 933 | 934 | return res 935 | 936 | 937 | def install(self): 938 | # use the uberenv package to trigger the right builds 939 | # and build an host-config.cmake file 940 | debug = "" 941 | if self.opts["spack_debug"]: 942 | debug = "--debug --stacktrace " 943 | 944 | if not self.use_install: 945 | install_cmd = "{0} {1}".format(self.spack_exe_path(), debug) 946 | if self.opts["ignore_ssl_errors"]: 947 | install_cmd += "-k " 948 | # build mode -- install path 949 | if self.build_mode == "install": 950 | install_cmd += "install " 951 | install_cmd = self.add_concretizer_opts(install_cmd) 952 | if self.opts["spack_allow_deprecated"]: 953 | install_cmd += "--deprecated " 954 | if self.opts["run_tests"]: 955 | install_cmd += "--test=root " 956 | if self.pkg_final_phase: 957 | install_cmd += "-u {0} ".format(self.pkg_final_phase) 958 | # build mode - dev build path 959 | elif self.build_mode == "dev-build": 960 | # dev build path 961 | install_cmd += "dev-build " 962 | install_cmd = self.add_concretizer_opts(install_cmd) 963 | if self.opts["spack_allow_deprecated"]: 964 | install_cmd += "--deprecated " 965 | install_cmd += "--quiet -d {0} ".format(self.pkg_src_dir) 966 | if self.pkg_final_phase: 967 | install_cmd += "-u {0} ".format(self.pkg_final_phase) 968 | # build mode -- original fake package path 969 | elif self.build_mode == "uberenv-pkg": 970 | install_cmd += "install " 971 | if self.opts["spack_allow_deprecated"]: 972 | install_cmd += "--deprecated " 973 | install_cmd = self.add_concretizer_opts(install_cmd) 974 | if self.pkg_final_phase: 975 | install_cmd += "-u {0} ".format(self.pkg_final_phase) 976 | else: 977 | print("[ERROR: unsupported build mode: {0}]".format(self.build_mode)) 978 | return -1 979 | if self.opts["build_jobs"]: 980 | install_cmd += "-j {0} ".format(self.opts["build_jobs"]) 981 | # for all cases we use the pkg name and spec 982 | install_cmd += "'{0}{1}'".format(self.pkg_name,self.opts["spec"]) 983 | res = sexe(install_cmd, echo=True) 984 | if res != 0: 985 | print("[ERROR: failure of spack install/dev-build]") 986 | return res 987 | 988 | full_spec = self.read_spack_full_spec(self.pkg_name,self.opts["spec"]) 989 | if "spack_activate" in self.project_opts: 990 | print("[activating dependent packages]") 991 | pkg_names = self.project_opts["spack_activate"].keys() 992 | for pkg_name in pkg_names: 993 | pkg_spec_requirements = self.project_opts["spack_activate"][pkg_name] 994 | activate=True 995 | for req in pkg_spec_requirements: 996 | if req not in full_spec: 997 | activate=False 998 | break 999 | if activate: 1000 | # Note: -f disables protection against same-named files 1001 | # blocking activation. We have to use this to avoid 1002 | # issues with python builtin vs external setuptools, etc 1003 | activate_cmd = "{0} activate -f ".format(self.spack_exe_path()) + pkg_name 1004 | res = sexe(activate_cmd, echo=True) 1005 | if res != 0: 1006 | return res 1007 | print("[done activating dependent packages]") 1008 | # note: this assumes package extends python when +python 1009 | # this may fail general cases 1010 | if self.build_mode == "install" and "+python" in full_spec: 1011 | activate_cmd = "{0} activate /{1}".format(self.spack_exe_path(), self.spec_hash) 1012 | res = sexe(activate_cmd, echo=True) 1013 | if res != 0: 1014 | return res 1015 | # when using install or uberenv-pkg mode, create a symlink to the host config 1016 | if self.build_mode == "install" or \ 1017 | self.build_mode == "uberenv-pkg" \ 1018 | or self.use_install: 1019 | # only create a symlink if you're completing all phases 1020 | if self.pkg_final_phase == None or self.pkg_final_phase == "install": 1021 | # use spec_hash to locate b/c other helper won't work if complex 1022 | # deps are provided in the spec (e.g: @ver+variant ^package+variant) 1023 | pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name, self.spec_hash) 1024 | if self.pkg_name != pkg_path["name"]: 1025 | print("[ERROR: Could not find install of {0} with hash {1}]".format(self.pkg_name,self.spec_hash)) 1026 | return -1 1027 | else: 1028 | # Symlink host-config file 1029 | hc_glob = glob.glob(pjoin(pkg_path["path"],"*.cmake")) 1030 | if len(hc_glob) > 0: 1031 | hc_path = hc_glob[0] 1032 | hc_fname = os.path.split(hc_path)[1] 1033 | if os.path.islink(hc_fname): 1034 | os.unlink(hc_fname) 1035 | elif os.path.isfile(hc_fname): 1036 | sexe("rm -f {0}".format(hc_fname)) 1037 | print("[symlinking host config file to {0}]".format(pjoin(self.dest_dir,hc_fname))) 1038 | os.symlink(hc_path,hc_fname) 1039 | # if user opt'd for an install, we want to symlink the final 1040 | # install to an easy place: 1041 | # Symlink install directory 1042 | if self.build_mode == "install": 1043 | pkg_lnk_dir = "{0}-install".format(self.pkg_name) 1044 | if os.path.islink(pkg_lnk_dir): 1045 | os.unlink(pkg_lnk_dir) 1046 | print("") 1047 | print("[symlinking install to {0}]".format(pjoin(self.dest_dir,pkg_lnk_dir))) 1048 | os.symlink(pkg_path["path"],pabs(pkg_lnk_dir)) 1049 | print("") 1050 | print("[install complete!]") 1051 | elif self.build_mode == "dev-build": 1052 | # we are in the "only dependencies" dev build case and the host-config 1053 | # file has to be copied from the do-be-deleted spack-build dir. 1054 | build_base = pjoin(self.dest_dir,"{0}-build".format(self.pkg_name)) 1055 | build_dir = pjoin(build_base,"spack-build") 1056 | pattern = "*{0}.cmake".format(self.pkg_name) 1057 | build_dir = pjoin(self.pkg_src_dir,"spack-build") 1058 | hc_glob = glob.glob(pjoin(build_dir,pattern)) 1059 | if len(hc_glob) > 0: 1060 | hc_path = hc_glob[0] 1061 | hc_fname = os.path.split(hc_path)[1] 1062 | if os.path.islink(hc_fname): 1063 | os.unlink(hc_fname) 1064 | print("[copying host config file to {0}]".format(pjoin(self.dest_dir,hc_fname))) 1065 | sexe("cp {0} {1}".format(hc_path,hc_fname)) 1066 | print("[removing project build directory {0}]".format(pjoin(build_dir))) 1067 | sexe("rm -rf {0}".format(build_dir)) 1068 | else: 1069 | print("[ERROR: Unsupported build mode {0}]".format(self.build_mode)) 1070 | return -1 1071 | 1072 | def get_mirror_path(self): 1073 | mirror_path = self.opts["mirror"] 1074 | if not mirror_path: 1075 | print("[--create-mirror requires a mirror directory]") 1076 | sys.exit(-1) 1077 | return mirror_path 1078 | 1079 | def create_mirror(self): 1080 | """ 1081 | Creates a spack mirror for pkg_name at mirror_path. 1082 | """ 1083 | 1084 | mirror_path = self.get_mirror_path() 1085 | 1086 | mirror_cmd = "{0} ".format(self.spack_exe_path()) 1087 | if self.opts["ignore_ssl_errors"]: 1088 | mirror_cmd += "-k " 1089 | mirror_cmd += "mirror create -d {0} --dependencies '{1}{2}'".format(mirror_path, 1090 | self.pkg_name, 1091 | self.opts["spec"]) 1092 | return sexe(mirror_cmd, echo=True) 1093 | 1094 | def find_spack_mirror(self, mirror_name): 1095 | """ 1096 | Returns the path of a defaults scoped spack mirror with the 1097 | given name, or None if no mirror exists. 1098 | """ 1099 | res, out = sexe("{0} mirror list".format(self.spack_exe_path()), ret_output=True) 1100 | mirror_path = None 1101 | for mirror in out.split('\n'): 1102 | if mirror: 1103 | parts = mirror.split() 1104 | if parts[0] == mirror_name: 1105 | mirror_path = parts[1] 1106 | return mirror_path 1107 | 1108 | def use_mirror(self): 1109 | """ 1110 | Configures spack to use mirror at a given path. 1111 | """ 1112 | mirror_name = self.pkg_name 1113 | mirror_path = self.get_mirror_path() 1114 | existing_mirror_path = self.find_spack_mirror(mirror_name) 1115 | 1116 | if existing_mirror_path and mirror_path != existing_mirror_path: 1117 | # Existing mirror has different URL, error out 1118 | print("[removing existing spack mirror `{0}` @ {1}]".format(mirror_name, 1119 | existing_mirror_path)) 1120 | # 1121 | # Note: In this case, spack says it removes the mirror, but we still 1122 | # get errors when we try to add a new one, sounds like a bug 1123 | # 1124 | sexe("{0} mirror remove --scope=defaults {1} ".format(self.spack_exe_path(), mirror_name), 1125 | echo=True) 1126 | existing_mirror_path = None 1127 | if not existing_mirror_path: 1128 | # Add if not already there 1129 | sexe("{0} mirror add --scope=defaults {1} {2}".format( 1130 | self.spack_exe_path(), mirror_name, mirror_path), echo=True) 1131 | print("[using mirror {0}]".format(mirror_path)) 1132 | 1133 | def find_spack_upstream(self, upstream_name): 1134 | """ 1135 | Returns the path of a defaults scoped spack upstream with the 1136 | given name, or None if no upstream exists. 1137 | """ 1138 | upstream_path = None 1139 | 1140 | res, out = sexe('{0} config get upstreams'.format(self.spack_exe_path()), ret_output=True) 1141 | if (not out) and ("upstreams:" in out): 1142 | out = out.replace(' ', '') 1143 | out = out.replace('install_tree:', '') 1144 | out = out.replace(':', '') 1145 | out = out.splitlines() 1146 | out = out[1:] 1147 | upstreams = dict(zip(out[::2], out[1::2])) 1148 | 1149 | for name in upstreams.keys(): 1150 | if name == upstream_name: 1151 | upstream_path = upstreams[name] 1152 | 1153 | return upstream_path 1154 | 1155 | def use_spack_upstream(self): 1156 | """ 1157 | Configures spack to use upstream at a given path. 1158 | """ 1159 | upstream_path = self.opts["upstream"] 1160 | if not upstream_path: 1161 | print("[--create-upstream requires a upstream directory]") 1162 | sys.exit(-1) 1163 | upstream_path = pabs(upstream_path) 1164 | upstream_name = self.pkg_name 1165 | existing_upstream_path = self.find_spack_upstream(upstream_name) 1166 | if (not existing_upstream_path) or (upstream_path != pabs(existing_upstream_path)): 1167 | # Existing upstream has different URL, error out 1168 | print("[removing existing spack upstream configuration file]") 1169 | sexe("rm spack/etc/spack/defaults/upstreams.yaml") 1170 | with open('spack/etc/spack/defaults/upstreams.yaml','w+') as upstreams_cfg_file: 1171 | upstreams_cfg_file.write("upstreams:\n") 1172 | upstreams_cfg_file.write(" {0}:\n".format(upstream_name)) 1173 | upstreams_cfg_file.write(" install_tree: {0}\n".format(upstream_path)) 1174 | 1175 | def setup_clingo(self): 1176 | """ 1177 | Attempts to install the clingo answer set programming library 1178 | if it is not already available as a Python module 1179 | """ 1180 | try: 1181 | import clingo 1182 | except ImportError: 1183 | import pip 1184 | pip_ver = pip.__version__ 1185 | # Requirement comes from https://github.com/pypa/manylinux 1186 | # JBE: I think the string comparison is somewhat correct here, if not we'll 1187 | # need to install setuptools for 'packaging.version' 1188 | if pip_ver < "19.3": 1189 | print("[!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") 1190 | print(" ERROR: pip version {0} is too old to install clingo".format(pip_ver)) 1191 | print(" pip 19.3 is required for PEP 599 support") 1192 | print(" Try running the following command to upgrade pip:") 1193 | print(" python3 -m pip install --user --upgrade pip") 1194 | print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!]") 1195 | sys.exit(1) 1196 | py_interp = sys.executable 1197 | clingo_pkg = "clingo" 1198 | uninstall_cmd = "{0} -m pip uninstall -y {1}".format(py_interp, clingo_pkg) 1199 | # Uninstall it first in case the available version failed due to differing arch 1200 | # pip will still return 0 in the case of a "trivial" uninstall 1201 | res = sexe(uninstall_cmd, echo=True) 1202 | if res != 0: 1203 | print("[ERROR: clingo uninstall failed with returncode {0}]".format(res)) 1204 | sys.exit(1) 1205 | install_cmd = "{0} -m pip install --user {1}".format(py_interp, clingo_pkg) 1206 | res = sexe(install_cmd, echo=True) 1207 | if res != 0: 1208 | print("[ERROR: clingo install failed with returncode {0}]".format(res)) 1209 | sys.exit(1) 1210 | 1211 | 1212 | def find_osx_sdks(): 1213 | """ 1214 | Finds installed osx sdks, returns dict mapping version to file system path 1215 | """ 1216 | res = {} 1217 | sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") 1218 | for sdk in sdks: 1219 | sdk_base = os.path.split(sdk)[1] 1220 | ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] 1221 | res[ver] = sdk 1222 | return res 1223 | 1224 | def setup_osx_sdk_env_vars(): 1225 | """ 1226 | Finds installed osx sdks, returns dict mapping version to file system path 1227 | """ 1228 | # find current osx version (10.11.6) 1229 | dep_tgt = platform.mac_ver()[0] 1230 | # sdk file names use short version (ex: 10.11) 1231 | dep_tgt_short = dep_tgt[:dep_tgt.rfind(".")] 1232 | # find installed sdks, ideally we want the sdk that matches the current os 1233 | sdk_root = None 1234 | sdks = find_osx_sdks() 1235 | if dep_tgt_short in sdks.keys(): 1236 | # matches our osx, use this one 1237 | sdk_root = sdks[dep_tgt_short] 1238 | elif len(sdks) > 0: 1239 | # for now, choose first one: 1240 | dep_tgt = sdks.keys()[0] 1241 | sdk_root = sdks[dep_tgt] 1242 | else: 1243 | # no valid sdks, error out 1244 | print("[ERROR: Could not find OSX SDK @ /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/]") 1245 | sys.exit(-1) 1246 | 1247 | env["MACOSX_DEPLOYMENT_TARGET"] = dep_tgt 1248 | env["SDKROOT"] = sdk_root 1249 | print("[setting MACOSX_DEPLOYMENT_TARGET to {0}]".format(env["MACOSX_DEPLOYMENT_TARGET"])) 1250 | print("[setting SDKROOT to {0}]".format(env[ "SDKROOT"])) 1251 | 1252 | 1253 | def print_uberenv_python_info(): 1254 | print("[uberenv python: {0}]".format(sys.executable)) 1255 | 1256 | 1257 | def main(): 1258 | """ 1259 | Clones and runs a package manager to setup third_party libs. 1260 | Also creates a host-config.cmake file that can be used by our project. 1261 | """ 1262 | 1263 | print_uberenv_python_info() 1264 | 1265 | # parse args from command line 1266 | opts, extra_opts = parse_args() 1267 | 1268 | # project options 1269 | opts["project_json"] = find_project_config(opts) 1270 | 1271 | # Initialize the environment -- use vcpkg on windows, spack otherwise 1272 | env = SpackEnv(opts, extra_opts) if not is_windows() else VcpkgEnv(opts, extra_opts) 1273 | 1274 | # Setup the necessary paths and directories 1275 | env.setup_paths_and_dirs() 1276 | 1277 | # Skip creating and setting up environment if env given 1278 | if not opts["skip_setup"]: 1279 | # Clone the package manager 1280 | env.clone_repo() 1281 | 1282 | os.chdir(env.dest_dir) 1283 | 1284 | # Patch the package manager, as necessary 1285 | env.patch() 1286 | 1287 | # Clean the build 1288 | env.clean_build() 1289 | 1290 | # Allow to end uberenv after spack is ready 1291 | if opts["setup_only"]: 1292 | 1293 | if not is_windows() and opts["upstream"] is not None: 1294 | env.use_spack_upstream() 1295 | 1296 | return 0 1297 | else: 1298 | os.chdir(env.dest_dir) 1299 | 1300 | 1301 | # Show the spec for what will be built 1302 | env.show_info() 1303 | 1304 | 1305 | ########################################################### 1306 | # we now have an instance of our package manager configured 1307 | # how we need it to build our tpls. At this point there are 1308 | # two possible next steps: 1309 | # 1310 | # *) create a mirror of the packages 1311 | # OR 1312 | # *) build 1313 | # 1314 | ########################################################### 1315 | if opts["create_mirror"]: 1316 | return env.create_mirror() 1317 | else: 1318 | if opts["mirror"] is not None: 1319 | env.use_mirror() 1320 | 1321 | if not is_windows() and opts["upstream"] is not None: 1322 | env.use_spack_upstream() 1323 | 1324 | res = env.install() 1325 | 1326 | return res 1327 | 1328 | if __name__ == "__main__": 1329 | sys.exit(main()) 1330 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | docutils 2 | sphinx==6.2.1 3 | sphinx-rtd-theme==1.2.2 4 | -------------------------------------------------------------------------------- /docs/sphinx/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | ############################################################################### 4 | # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 5 | # 6 | # Produced at the Lawrence Livermore National Laboratory 7 | # 8 | # LLNL-CODE-666778 9 | # 10 | # All rights reserved. 11 | # 12 | # This file is part of Conduit. 13 | # 14 | # For details, see: http://software.llnl.gov/conduit/. 15 | # 16 | # Please also read conduit/LICENSE 17 | # 18 | # Redistribution and use in source and binary forms, with or without 19 | # modification, are permitted provided that the following conditions are met: 20 | # 21 | # * Redistributions of source code must retain the above copyright notice, 22 | # this list of conditions and the disclaimer below. 23 | # 24 | # * Redistributions in binary form must reproduce the above copyright notice, 25 | # this list of conditions and the disclaimer (as noted below) in the 26 | # documentation and/or other materials provided with the distribution. 27 | # 28 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 29 | # be used to endorse or promote products derived from this software without 30 | # specific prior written permission. 31 | # 32 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 33 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 34 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 35 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 36 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 37 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 41 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 42 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 43 | # POSSIBILITY OF SUCH DAMAGE. 44 | # 45 | ############################################################################### 46 | # 47 | # Uberenv documentation build configuration file, created by 48 | # sphinx-quickstart on Thu Oct 16 11:23:46 2014. 49 | # 50 | # This file is execfile()d with the current directory set to its 51 | # containing dir. 52 | # 53 | # Note that not all possible configuration values are present in this 54 | # autogenerated file. 55 | # 56 | # All configuration values have a default; values that are commented out 57 | # serve to show the default. 58 | 59 | import sys 60 | import os 61 | 62 | # If extensions (or modules to document with autodoc) are in another directory, 63 | # add these directories to sys.path here. If the directory is relative to the 64 | # documentation root, use os.path.abspath to make it absolute, like shown here. 65 | #sys.path.insert(0, os.path.abspath('.')) 66 | 67 | # -- General configuration ------------------------------------------------ 68 | 69 | # If your documentation needs a minimal Sphinx version, state it here. 70 | #needs_sphinx = '1.0' 71 | 72 | # Add any Sphinx extension module names here, as strings. They can be 73 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 74 | # ones. 75 | extensions = [ 76 | 'sphinx.ext.autodoc', 77 | 'sphinx.ext.doctest', 78 | 'sphinx.ext.todo', 79 | 'sphinx.ext.coverage', 80 | 'sphinx.ext.mathjax' 81 | ] 82 | 83 | # try to add the breathe extension 84 | try: 85 | import breathe 86 | extensions.append('breathe') 87 | except: 88 | pass 89 | 90 | # Add any paths that contain templates here, relative to this directory. 91 | # templates_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_templates'] 92 | 93 | # The suffix of source filenames. 94 | source_suffix = '.rst' 95 | 96 | # The encoding of source files. 97 | #source_encoding = 'utf-8-sig' 98 | 99 | # The master toctree document. 100 | master_doc = 'index' 101 | 102 | # General information about the project. 103 | project = u'Uberenv' 104 | copyright = u'Copyright (c) 2014-2025, LLNS' 105 | 106 | # The version info for the project you're documenting, acts as replacement for 107 | # |version| and |release|, also used in various other places throughout the 108 | # built documents. 109 | # 110 | # The short X.Y version. 111 | version = 'current' 112 | # The full version, including alpha/beta/rc tags. 113 | release = 'current' 114 | 115 | # The language for content autogenerated by Sphinx. Refer to documentation 116 | # for a list of supported languages. 117 | #language = None 118 | 119 | # There are two options for replacing |today|: either, you set today to some 120 | # non-false value, then it is used: 121 | #today = '' 122 | # Else, today_fmt is used as the format for a strftime call. 123 | #today_fmt = '%B %d, %Y' 124 | 125 | # List of patterns, relative to source directory, that match files and 126 | # directories to ignore when looking for source files. 127 | exclude_patterns = ['_build'] 128 | 129 | # The reST default role (used for this markup: `text`) to use for all 130 | # documents. 131 | #default_role = None 132 | 133 | # If true, '()' will be appended to :func: etc. cross-reference text. 134 | #add_function_parentheses = True 135 | 136 | # If true, the current module name will be prepended to all description 137 | # unit titles (such as .. function::). 138 | #add_module_names = True 139 | 140 | # If true, sectionauthor and moduleauthor directives will be shown in the 141 | # output. They are ignored by default. 142 | #show_authors = False 143 | 144 | # The name of the Pygments (syntax highlighting) style to use. 145 | pygments_style = 'sphinx' 146 | 147 | # A list of ignored prefixes for module index sorting. 148 | #modindex_common_prefix = [] 149 | 150 | # If true, keep warnings as "system message" paragraphs in the built documents. 151 | #keep_warnings = False 152 | 153 | 154 | # -- Options for HTML output ---------------------------------------------- 155 | 156 | # The theme to use for HTML and HTML Help pages. See the documentation for 157 | # a list of builtin themes. 158 | html_theme = 'sphinx_rtd_theme' 159 | 160 | # Theme options are theme-specific and customize the look and feel of a theme 161 | # further. For a list of options available for each theme, see the 162 | # documentation. 163 | html_theme_options = { 'logo_only' : True } 164 | 165 | # Add any paths that contain custom themes here, relative to this directory. 166 | #html_theme_path = [] 167 | 168 | # The name for this set of Sphinx documents. If None, it defaults to 169 | # " v documentation". 170 | #html_title = None 171 | 172 | # A shorter title for the navigation bar. Default is the same as html_title. 173 | #html_short_title = None 174 | 175 | # The name of an image file (relative to this directory) to place at the top 176 | # of the sidebar. 177 | #html_logo = 178 | 179 | # The name of an image file (within the static path) to use as favicon of the 180 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 181 | # pixels large. 182 | #html_favicon = None 183 | 184 | # Add any paths that contain custom static files (such as style sheets) here, 185 | # relative to this directory. They are copied after the builtin static files, 186 | # so a file named "default.css" will overwrite the builtin "default.css". 187 | # html_static_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_static'] 188 | 189 | # Add any extra paths that contain custom files (such as robots.txt or 190 | # .htaccess) here, relative to this directory. These files are copied 191 | # directly to the root of the documentation. 192 | #html_extra_path = [] 193 | 194 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 195 | # using the given strftime format. 196 | #html_last_updated_fmt = '%b %d, %Y' 197 | 198 | # If true, SmartyPants will be used to convert quotes and dashes to 199 | # typographically correct entities. 200 | #html_use_smartypants = True 201 | 202 | # Custom sidebar templates, maps document names to template names. 203 | #html_sidebars = {} 204 | 205 | # Additional templates that should be rendered to pages, maps page names to 206 | # template names. 207 | #html_additional_pages = {} 208 | 209 | # If false, no module index is generated. 210 | #html_domain_indices = True 211 | 212 | # If false, no index is generated. 213 | #html_use_index = True 214 | 215 | # If true, the index is split into individual pages for each letter. 216 | #html_split_index = False 217 | 218 | # If true, links to the reST sources are added to the pages. 219 | #html_show_sourcelink = True 220 | 221 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 222 | #html_show_sphinx = True 223 | 224 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 225 | #html_show_copyright = True 226 | 227 | # If true, an OpenSearch description file will be output, and all pages will 228 | # contain a tag referring to it. The value of this option must be the 229 | # base URL from which the finished HTML is served. 230 | #html_use_opensearch = '' 231 | 232 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 233 | #html_file_suffix = None 234 | 235 | # Output file base name for HTML help builder. 236 | htmlhelp_basename = 'Uberenvdoc' 237 | 238 | 239 | # -- Options for LaTeX output --------------------------------------------- 240 | 241 | latex_elements = { 242 | # The paper size ('letterpaper' or 'a4paper'). 243 | #'papersize': 'letterpaper', 244 | 245 | # The font size ('10pt', '11pt' or '12pt'). 246 | #'pointsize': '10pt', 247 | 248 | # Additional stuff for the LaTeX preamble. 249 | #'preamble': '', 250 | } 251 | 252 | # Grouping the document tree into LaTeX files. List of tuples 253 | # (source start file, target name, title, 254 | # author, documentclass [howto, manual, or own class]). 255 | latex_documents = [ 256 | ('index', 'Uberenv.tex', u'Uberenv Documentation', 257 | u'LLNS', 'manual'), 258 | ] 259 | 260 | # The name of an image file (relative to this directory) to place at the top of 261 | # the title page. 262 | #latex_logo = None 263 | 264 | # For "manual" documents, if this is true, then toplevel headings are parts, 265 | # not chapters. 266 | #latex_use_parts = False 267 | 268 | # If true, show page references after internal links. 269 | #latex_show_pagerefs = False 270 | 271 | # If true, show URL addresses after external links. 272 | #latex_show_urls = False 273 | 274 | # Documents to append as an appendix to all manuals. 275 | #latex_appendices = [] 276 | 277 | # If false, no module index is generated. 278 | #latex_domain_indices = True 279 | 280 | 281 | # -- Options for manual page output --------------------------------------- 282 | 283 | # One entry per manual page. List of tuples 284 | # (source start file, name, description, authors, manual section). 285 | man_pages = [ 286 | ('index', 'uberenv', u'Uberenv Documentation', 287 | [u'LLNS'], 1) 288 | ] 289 | 290 | # If true, show URL addresses after external links. 291 | #man_show_urls = False 292 | 293 | 294 | # -- Options for Texinfo output ------------------------------------------- 295 | 296 | # Grouping the document tree into Texinfo files. List of tuples 297 | # (source start file, target name, title, author, 298 | # dir menu entry, description, category) 299 | texinfo_documents = [ 300 | ('index', 'Uberenv', u'Uberenv Documentation', 301 | u'LLNS', 'Uberenv', 'Automates using spack to build and deploy software.', 302 | 'Miscellaneous'), 303 | ] 304 | 305 | # Documents to append as an appendix to all manuals. 306 | #texinfo_appendices = [] 307 | 308 | # If false, no module index is generated. 309 | #texinfo_domain_indices = True 310 | 311 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 312 | #texinfo_show_urls = 'footnote' 313 | 314 | # If true, do not generate a @detailmenu in the "Top" node's menu. 315 | #texinfo_no_detailmenu = False 316 | 317 | 318 | # try to use the read the docs theme 319 | try: 320 | import sphinx_rtd_theme 321 | html_theme = "sphinx_rtd_theme" 322 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 323 | except: 324 | pass 325 | -------------------------------------------------------------------------------- /docs/sphinx/index.rst: -------------------------------------------------------------------------------- 1 | .. ############################################################################ 2 | .. # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 3 | .. # 4 | .. # Produced at the Lawrence Livermore National Laboratory 5 | .. # 6 | .. # LLNL-CODE-666778 7 | .. # 8 | .. # All rights reserved. 9 | .. # 10 | .. # This file is part of Conduit. 11 | .. # 12 | .. # For details, see: http://software.llnl.gov/conduit/. 13 | .. # 14 | .. # Please also read conduit/LICENSE 15 | .. # 16 | .. # Redistribution and use in source and binary forms, with or without 17 | .. # modification, are permitted provided that the following conditions are met: 18 | .. # 19 | .. # * Redistributions of source code must retain the above copyright notice, 20 | .. # this list of conditions and the disclaimer below. 21 | .. # 22 | .. # * Redistributions in binary form must reproduce the above copyright notice, 23 | .. # this list of conditions and the disclaimer (as noted below) in the 24 | .. # documentation and/or other materials provided with the distribution. 25 | .. # 26 | .. # * Neither the name of the LLNS/LLNL nor the names of its contributors may 27 | .. # be used to endorse or promote products derived from this software without 28 | .. # specific prior written permission. 29 | .. # 30 | .. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 31 | .. # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 32 | .. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 | .. # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 34 | .. # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 35 | .. # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 | .. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 | .. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 | .. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 | .. # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 40 | .. # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 41 | .. # POSSIBILITY OF SUCH DAMAGE. 42 | .. # 43 | .. ############################################################################ 44 | 45 | .. _building_with_uberenv: 46 | 47 | Uberenv 48 | ~~~~~~~ 49 | 50 | **Uberenv** automates using a package manager to build and deploy software. 51 | It uses `Spack `_ on Unix-based systems (e.g. Linux and macOS) 52 | and `Vcpkg `_ on Windows systems. 53 | 54 | Many projects leverage package managers, like Spack and Vcpkg, to help build the software dependencies needed to 55 | develop and deploy their projects on HPC systems. Uberenv is a python script that helps automate the usage of a package manager to build 56 | third-party dependencies for development and deployment. 57 | 58 | Uberenv was released as part of Conduit (https://github.com/LLNL/conduit/). It is included in-source in several projects. The 59 | https://github.com/llnl/uberenv/ repo is used to hold the latest reference version of Uberenv. 60 | 61 | Several projects are using Uberenv for Continuous Integration (CI) purposes. The process is documented `here `_. 62 | 63 | uberenv.py 64 | ~~~~~~~~~~ 65 | 66 | Uberenv is a single file python script (``uberenv.py``) that automates fetching Spack or Vcpkg, building and installing third party dependencies, 67 | and can optionally install top-level packages as well. To automate the full install process, Uberenv uses a target Spack or Vcpkg 68 | package along with extra settings such as compilers and external third party package details for common HPC platforms. 69 | 70 | Uberenv is included directly in a project's source code repo, usually in the folder: ``scripts/uberenv/``. 71 | This folder is also used to store extra configuration files unique to the target project. 72 | Uberenv uses a ``project.json`` file to specify project details, including the target package name 73 | and the base branch or commit in the package manager. 74 | 75 | Conduit's source repo serves as an example for Uberenv and Spack configuration files, etc: 76 | 77 | https://github.com/LLNL/conduit/tree/master/scripts/uberenv 78 | 79 | Uberenv can also be used as a submodule of the user project, where one must provide a configuration file named 80 | ``.uberenv_config.json`` in a parent directory. This file is similar to ``project.json`` in purpose, but should 81 | additionally provide the entries ``spack_configs_path`` and ``spack_packages_path``. 82 | See :ref:`project_configuration` for more details. 83 | 84 | .. Note:: 85 | Uberenv requires python 3.3 or above. 86 | 87 | Uberenv is developed by LLNL, originally in support of the `Ascent `_, 88 | `Axom `_, and `Conduit `_ projects. It is now also used 89 | by `Umpire `_, `CHAI `_, `RAJA `_ 90 | and `Serac `_, among others. 91 | 92 | 93 | Command Line Options 94 | ~~~~~~~~~~~~~~~~~~~~ 95 | 96 | Build Configuration 97 | ------------------- 98 | 99 | Uberenv has a few options that allow you to control how dependencies are built: 100 | 101 | =========================== ============================================== ================================================= 102 | Option Description Default 103 | =========================== ============================================== ================================================= 104 | ``--prefix`` Destination directory ``uberenv_libs`` 105 | ``--spec`` Spack spec without preceding package name linux: **%gcc** 106 | osx: **%clang** 107 | ``--spack-env-name`` The name of the created Spack Environment ``spack_env`` 108 | ``--spack-env-file`` Path to Spack Environment config See :ref:`spack_configs` 109 | (e.g. spack.yaml) 110 | ``--spack-build-mode`` Mode used to build third party dependencies ``dev-build`` 111 | ``--spack-debug`` Enable Spack debug mode for all commands **none** (False) 112 | ``-k`` Ignore SSL Errors **False** 113 | ``--install`` Fully install target, not just dependencies **False** 114 | ``--run_tests`` Invoke tests during build and against install **False** 115 | ``--setup-only`` Only download and setup Spack **False** 116 | ``--skip-setup`` Only install (using pre-setup Spack) **False** 117 | ``--setup-and-env-only`` Download, setup Spack, and generate env file **False** 118 | ``--skip-setup-and-env`` Only install (using pre-setup Spack/env file) **False** 119 | ``--spack-externals`` Space delimited string of packages for **none** 120 | Spack to search for externals 121 | ``--spack-compiler-paths`` Space delimited string of paths for **none** 122 | Spack to search for compilers 123 | ``--project-json`` File for project specific settings See :ref:`project_configuration` 124 | ``--triplet`` (vcpkg) Target architecture and linkage ``VCPKG_DEFAULT_TRIPLET`` environment variable, 125 | if present, ``x86-Windows`` otherwise 126 | =========================== ============================================== ================================================= 127 | 128 | The ``--spack-env-name`` will be created in path specified by ``--prefix``. 129 | 130 | The ``-k`` option exists for sites where SSL certificate interception undermines fetching 131 | from github and https hosted source tarballs. When enabled, Uberenv clones Spack using: 132 | 133 | ``git -c http.sslVerify=false clone https://github.com/llnl/spack.git`` 134 | 135 | And passes ``-k`` to any Spack commands that may fetch via https. 136 | 137 | 138 | Default invocations: 139 | 140 | **Linux** 141 | 142 | ``python scripts/uberenv/uberenv.py --prefix uberenv_libs --spec %gcc`` 143 | 144 | **OSX** 145 | 146 | ``python scripts/uberenv/uberenv.py --prefix uberenv_libs --spec %clang`` 147 | 148 | **Windows** 149 | 150 | ``python scripts/uberenv/uberenv.py --prefix uberenv_libs --triplet x86-windows`` 151 | 152 | See `Vcpkg user docs `_ for more information about triplets. 153 | 154 | Use the ``--install`` option to install the target package (not just its development dependencies): 155 | 156 | ``python scripts/uberenv/uberenv.py --install`` 157 | 158 | 159 | If the target Spack package supports Spack's testing hooks, you can run tests during the build process to validate the build and install, using the ``--run_tests`` option: 160 | 161 | ``python scripts/uberenv/uberenv.py --install --run_tests`` 162 | 163 | For details on Spack's spec syntax, see the `Spack Specs & dependencies `_ documentation. 164 | 165 | .. _spack_configs: 166 | 167 | Spack Configurations 168 | -------------------- 169 | 170 | Uberenv looks for the ``spack.yaml`` configuration file, also known as an Environment file, under ``scripts/uberenv/spack_configs/{platform}`` or 171 | ``{spack_config_paths}/{platform}``, where: ``{platform}`` must match the platform determined by Uberenv (``SYS_TYPE`` on LC and ``darwin`` on 172 | OSX). ``{spack_configs_path}`` can be specified in the json config file. 173 | 174 | You may instead use the ``--spack-env-file`` option to enforce the use of a specific Spack Environment file. This file 175 | does not need to be called ``spack.yaml`` if you wish to call it something else, like according to its platform for 176 | example. See the `Spack Environments (spack.yaml) `_ 177 | documentation for details. 178 | 179 | If an Environment file cannot be found, Uberenv will generate one and copy it to ``{package_source_dir}/spack.yaml``. 180 | Spack will find packages and compilers on its own based on ``--spack-externals`` and ``--spack-compiler-paths``. If 181 | these options are not specified either on the command line or project json, Spack will find all compilers and packages 182 | it can. To prevent Uberenv from creating an Environment file in future builds, specify your ``--spack-environment-file`` 183 | to the one generated. 184 | 185 | When run, ``uberenv.py`` check outs a specific version of Spack from github as ``spack`` in the 186 | destination directory. It then uses Spack to build and install the target packages' dependencies into 187 | ``spack/opt/spack/``. Finally, the target package generates a host-config file ``{hostname}.cmake``, which is 188 | copied to destination directory. This file specifies the compiler settings and paths to all of the dependencies. 189 | 190 | .. note:: 191 | Instead of two yaml files (``packages.yaml`` and ``compilers.yaml``), Ubernev uses a single ``spack.yaml``, which is 192 | simply the combination of the original two under ``spack:``. 193 | 194 | .. code-block:: yaml 195 | 196 | spack: 197 | # contents of packages.yaml 198 | # contents of compilers.yaml 199 | 200 | .. _project_configuration: 201 | 202 | Project Configuration 203 | --------------------- 204 | 205 | Project level configuration options can also be addressed using a json file and some settings can be overridden on command line. This json file 206 | is found in the in the following order: 207 | 208 | 1. ``--project-json=[path/to/project.json]`` command line option 209 | 2. ``project.json`` that lives in the same directory as ``uberenv.py`` 210 | 3. ``.uberenv_config.json`` found recursively in a parent directory (typically at the root of your project) 211 | 212 | Project settings are as follows: 213 | 214 | ========================= ========================== ================================================ ======================================= 215 | Setting Command line Option Description Default 216 | ========================= ========================== ================================================ ======================================= 217 | package_name ``--package-name`` Spack package name **None** 218 | package_version **None** Spack package version **None** 219 | package_final_phase ``--package-final-phase`` Controls after which phase Spack should stop **None** 220 | package_source_dir ``--package-source-dir`` Controls the source directory Spack should use **None** 221 | force_commandline_prefix **None** Force user to specify `--prefix` on command line ``false`` 222 | spack_url **None** Download url for Spack ``https://github.com/spack/spack.git`` 223 | spack_commit **None** Spack commit to checkout **None** 224 | spack_activate **None** Spack packages to activate **None** 225 | spack_build_mode ``--spack-build-mode`` Set mode used to build TPLs with Spack ``dev-build`` 226 | spack_configs_path **None** Directory with Spack configs to be autodetected ``spack_configs`` 227 | spack_packages_path **None** Directory|List with Package Repos to be added ``packages`` 228 | spack_setup_clingo **None** Do not install clingo if set to ``false`` **None** 229 | spack_externals ``--spack-externals`` Space delimited string of packages for Spack to **None** 230 | search for externals 231 | spack_compiler_paths ``--spack-compiler-paths`` Space delimited string of paths for Spack to **None** 232 | search for compilers 233 | vcpkg_url **None** Download url for Vcpkg ``https://github.com/microsoft/vcpkg`` 234 | vcpkg_branch **None** Vcpkg branch to checkout ``master`` 235 | vcpkg_commit **None** Vcpkg commit to checkout **None** 236 | vcpkg_ports_path ``--vcpkg-ports-path`` Folder with vcpkg ports files **None** 237 | ========================= ========================== ================================================ ======================================= 238 | 239 | If a ``spack_commit`` is present, it supercedes the ``spack_branch`` option, and similarly for ``vcpkg_commit``and ``vcpkg_branch``. 240 | 241 | When used as a submodule ``.uberenv_config.json`` should define both ``spack_configs_path`` and ``spack_packages_path``, 242 | providing Uberenv with the respective location of ``spack_configs`` and ``packages`` directories. 243 | Note that they cannot sit next to ``uberenv.py``, since by default, the Uberenv repo does not provide them. 244 | 245 | ``spack_packages_path`` can either be a singular directory or a list of directories. These are relative to the 246 | location of the ``.uberenv_config.json``. When it is a list, the directories are added from left to right in Spack 247 | and right-most directories have the highest priority. The built-in Spack package repository is the lowest priority. 248 | Example: 249 | 250 | .. code-block:: json 251 | 252 | "spack_packages_path": "package/repo/higher/than/spacks", 253 | "spack_packages_path": ["package/repo/higher/than/spacks", "package/repo/even/higher"], 254 | 255 | .. note:: 256 | Uberenv no longer copies all directories that exist under ``spack_packages_path`` to the cloned 257 | Spack. A ``repo.yaml`` is now required in the previous directory of each packages path instead. 258 | Inside ``repo.yaml``, you only need a namespace, which can simply be the name of the package 259 | you're installing. See 260 | `Spack's documentation `_. 261 | 262 | .. note:: 263 | For an example of how to craft a ``project.json`` / ``.uberenv_config.json`` file a target project, 264 | see: `Axom's project.json file `_. 265 | 266 | Optimization 267 | ------------ 268 | 269 | Uberenv also features options to optimize the installation 270 | 271 | ===================== ============================================== ================================================ 272 | Option Description Default 273 | ===================== ============================================== ================================================ 274 | ``--mirror`` Location of a Spack mirror **None** 275 | ``--create-mirror`` Creates a Spack mirror at specified location **None** 276 | ``--upstream`` Location of a Spack upstream **None** 277 | ===================== ============================================== ================================================ 278 | 279 | .. note:: 280 | These options are only currently available for spack. 281 | -------------------------------------------------------------------------------- /gen_spack_env_script.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright (c) 2015-2024, Lawrence Livermore National Security, LLC. 3 | # 4 | # Produced at the Lawrence Livermore National Laboratory 5 | # 6 | # LLNL-CODE-716457 7 | # 8 | # All rights reserved. 9 | # 10 | # This file is part of Ascent. 11 | # 12 | # For details, see: http://ascent.readthedocs.io/. 13 | # 14 | # Please also read ascent/LICENSE 15 | # 16 | # Redistribution and use in source and binary forms, with or without 17 | # modification, are permitted provided that the following conditions are met: 18 | # 19 | # * Redistributions of source code must retain the above copyright notice, 20 | # this list of conditions and the disclaimer below. 21 | # 22 | # * Redistributions in binary form must reproduce the above copyright notice, 23 | # this list of conditions and the disclaimer (as noted below) in the 24 | # documentation and/or other materials provided with the distribution. 25 | # 26 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 27 | # be used to endorse or promote products derived from this software without 28 | # specific prior written permission. 29 | # 30 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 31 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 32 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 34 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 35 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 40 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 41 | # POSSIBILITY OF SUCH DAMAGE. 42 | # 43 | ############################################################################### 44 | import os 45 | import sys 46 | import subprocess 47 | 48 | from os.path import join as pjoin 49 | 50 | # if you have bad luck with spack load, this 51 | # script is for you! 52 | # 53 | # Looks for subdir: spack or uberenv_libs/spack 54 | # queries spack for given package names and 55 | # creates a bash script that adds those to your path 56 | # 57 | # 58 | # usage: 59 | # python gen_spack_env_script.py [spack_pkg_1 spack_pkg_2 ...] 60 | # 61 | 62 | def sexe(cmd,ret_output=False,echo = True): 63 | """ Helper for executing shell commands. """ 64 | if echo: 65 | print("[exe: {}]".format(cmd)) 66 | if ret_output: 67 | p = subprocess.Popen(cmd, 68 | shell=True, 69 | stdout=subprocess.PIPE, 70 | stderr=subprocess.STDOUT) 71 | res = p.communicate()[0] 72 | res = res.decode('utf8') 73 | return p.returncode,res 74 | else: 75 | return subprocess.call(cmd,shell=True) 76 | 77 | 78 | def spack_exe(spath=None): 79 | if spath is None: 80 | to_try = [pjoin("uberenv_libs","spack"), "spack"] 81 | for p in to_try: 82 | abs_p = os.path.abspath(p) 83 | print("[looking for spack directory at: {}]".format(abs_p)) 84 | if os.path.isdir(abs_p): 85 | print("[FOUND spack directory at: {}]".format(abs_p)) 86 | return os.path.abspath(pjoin(abs_p,"bin","spack")) 87 | print("[ERROR: failed to find spack directory!]") 88 | sys.exit(-1) 89 | else: 90 | spack_exe = os.path.abspath(spath,"bin","spack") 91 | if not os.path.isfile(spack_exec): 92 | print("[ERROR: failed to find spack directory at spath={}]").format(spath) 93 | sys.exit(-1) 94 | return spack_exe 95 | 96 | def find_pkg(pkg_name): 97 | r,rout = sexe(spack_exe() + " find -p " + pkg_name,ret_output = True) 98 | print(rout) 99 | for l in rout.split("\n"): 100 | print(l) 101 | lstrip = l.strip() 102 | if not lstrip == "" and \ 103 | not lstrip.startswith("==>") and \ 104 | not lstrip.startswith("--"): 105 | return {"name": pkg_name, "path": l.split()[-1]} 106 | print("[ERROR: failed to find package named '{}']".format(pkg_name)) 107 | sys.exit(-1) 108 | 109 | def path_cmd(pkg): 110 | return('export PATH={}:$PATH\n'.format((pjoin(pkg["path"],"bin")))) 111 | 112 | def write_env_script(pkgs): 113 | ofile = open("s_env.sh","w") 114 | for p in pkgs: 115 | print("[found {} at {}]".format(p["name"],p["path"])) 116 | ofile.write("# {}\n".format(p["name"])) 117 | ofile.write(path_cmd(p)) 118 | print("[created {}]".format(os.path.abspath("s_env.sh"))) 119 | 120 | def main(): 121 | pkgs = [find_pkg(pkg) for pkg in sys.argv[1:]] 122 | if len(pkgs) > 0: 123 | write_env_script(pkgs) 124 | else: 125 | print("usage: python gen_spack_env_script.py spack_pkg_1 spack_pkg_2 ...") 126 | 127 | if __name__ == "__main__": 128 | main() 129 | -------------------------------------------------------------------------------- /uberenv.py: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | "exec" "python3" "-u" "-B" "$0" "$@" 3 | ############################################################################### 4 | # Copyright (c) 2014-2025, Lawrence Livermore National Security, LLC. 5 | # 6 | # Produced at the Lawrence Livermore National Laboratory 7 | # 8 | # LLNL-CODE-666778 9 | # 10 | # All rights reserved. 11 | # 12 | # This file is part of Conduit. 13 | # 14 | # For details, see https://lc.llnl.gov/conduit/. 15 | # 16 | # Please also read conduit/LICENSE 17 | # 18 | # Redistribution and use in source and binary forms, with or without 19 | # modification, are permitted provided that the following conditions are met: 20 | # 21 | # * Redistributions of source code must retain the above copyright notice, 22 | # this list of conditions and the disclaimer below. 23 | # 24 | # * Redistributions in binary form must reproduce the above copyright notice, 25 | # this list of conditions and the disclaimer (as noted below) in the 26 | # documentation and/or other materials provided with the distribution. 27 | # 28 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 29 | # be used to endorse or promote products derived from this software without 30 | # specific prior written permission. 31 | # 32 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 33 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 34 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 35 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 36 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 37 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 41 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 42 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 43 | # POSSIBILITY OF SUCH DAMAGE. 44 | # 45 | ############################################################################### 46 | 47 | """ 48 | file: uberenv.py 49 | 50 | description: automates using a package manager to install a project. 51 | Uses spack on Unix-based systems and Vcpkg on Windows-based systems. 52 | 53 | """ 54 | 55 | import os 56 | import sys 57 | import subprocess 58 | import shutil 59 | import socket 60 | import platform 61 | import json 62 | import glob 63 | import re 64 | import argparse 65 | 66 | from functools import partial 67 | 68 | from os import environ as env 69 | from os.path import join as pjoin 70 | from os.path import abspath as pabs 71 | 72 | # Since we use subprocesses, flushing prints allows us to keep logs in 73 | # order. 74 | print = partial(print, flush=True) 75 | 76 | def sexe(cmd,ret_output=False,echo=False): 77 | """ Helper for executing shell commands. """ 78 | if echo: 79 | print("[exe: {0}]".format(cmd)) 80 | if ret_output: 81 | p = subprocess.Popen(cmd, 82 | shell=True, 83 | stdout=subprocess.PIPE, 84 | stderr=subprocess.STDOUT) 85 | out = p.communicate()[0] 86 | out = out.decode('utf8') 87 | return p.returncode,out 88 | else: 89 | return subprocess.call(cmd,shell=True) 90 | 91 | 92 | def parse_args(): 93 | "Parses args from command line" 94 | parser = argparse.ArgumentParser() 95 | parser.add_argument("--install", 96 | action="store_true", 97 | dest="install", 98 | default=False, 99 | help="Install `package_name`, not just its dependencies.") 100 | 101 | # where to install 102 | parser.add_argument("--prefix", 103 | dest="prefix", 104 | default=None, 105 | help="destination directory") 106 | 107 | # Spack spec without preceding package name 108 | parser.add_argument("--spec", 109 | dest="spec", 110 | default=None, 111 | help="Spack spec without preceding package name") 112 | 113 | # for vcpkg, what architecture to target 114 | parser.add_argument("--triplet", 115 | dest="vcpkg_triplet", 116 | default=None, 117 | help="vcpkg architecture triplet") 118 | 119 | # optional location of spack mirror 120 | parser.add_argument("--mirror", 121 | dest="mirror", 122 | default=None, 123 | help="spack mirror directory") 124 | 125 | # flag to create mirror 126 | parser.add_argument("--create-mirror", 127 | action="store_true", 128 | dest="create_mirror", 129 | default=False, 130 | help="Create spack mirror") 131 | 132 | # optional location of spack upstream 133 | parser.add_argument("--upstream", 134 | dest="upstream", 135 | default=None, 136 | help="add an external spack instance as upstream") 137 | 138 | # optional spack --reuse concretizer behaviour 139 | parser.add_argument("--reuse", 140 | dest="reuse", 141 | default=False, 142 | help="Use spack v0.17+ --reuse functionality for spec, install and dev-build.") 143 | 144 | # this option allows a user to set the directory for their vcpkg ports on Windows 145 | parser.add_argument("--vcpkg-ports-path", 146 | dest="vcpkg_ports_path", 147 | default=None, 148 | help="dir with vckpkg ports") 149 | 150 | # overrides package_name 151 | parser.add_argument("--package-name", 152 | dest="package_name", 153 | default=None, 154 | help="override the default package name") 155 | 156 | # uberenv spack tpl build mode 157 | parser.add_argument("--spack-build-mode", 158 | dest="spack_build_mode", 159 | default=None, 160 | help="set mode used to build third party dependencies with spack" 161 | "(options: 'dev-build' 'uberenv-pkg' 'install' " 162 | "[default: 'dev-build'] )\n") 163 | 164 | # spack debug mode 165 | parser.add_argument("--spack-debug", 166 | dest="spack_debug", 167 | action="store_true", 168 | default=False, 169 | help="add debug option to all spack commands") 170 | 171 | # spack allow deprecated packages 172 | parser.add_argument("--spack-allow-deprecated", 173 | dest="spack_allow_deprecated", 174 | action="store_true", 175 | default=False, 176 | help="add --deprecated to spack install commands") 177 | 178 | # controls after which package phase spack should stop 179 | parser.add_argument("--package-final-phase", 180 | dest="package_final_phase", 181 | default=None, 182 | help="override the default phase after which spack should stop") 183 | 184 | # controls source_dir spack should use to build the package 185 | parser.add_argument("--package-source-dir", 186 | dest="package_source_dir", 187 | default=None, 188 | help="override the default source dir spack should use") 189 | 190 | # a file that holds settings for a specific project 191 | # using uberenv.py 192 | parser.add_argument("--project-json", 193 | dest="project_json", 194 | default=pjoin(uberenv_script_dir(),"project.json"), 195 | help="uberenv project settings json file") 196 | 197 | # option to explicitly set the number of build jobs 198 | parser.add_argument("-j", 199 | dest="build_jobs", 200 | default=None, 201 | help="Explicitly set build jobs") 202 | 203 | # flag to use insecure curl + git 204 | parser.add_argument("-k", 205 | action="store_true", 206 | dest="ignore_ssl_errors", 207 | default=False, 208 | help="Ignore SSL Errors") 209 | 210 | # option to force a pull of the package manager 211 | parser.add_argument("--pull", 212 | action="store_true", 213 | dest="repo_pull", 214 | default=False, 215 | help="Pull from package manager, if repo already exists") 216 | 217 | # option to force for clean of packages specified to 218 | # be cleaned in the project.json 219 | parser.add_argument("--clean", 220 | action="store_true", 221 | dest="spack_clean", 222 | default=False, 223 | help="Force uninstall of packages specified in project.json") 224 | 225 | # option to tell spack to run tests 226 | parser.add_argument("--run_tests", 227 | action="store_true", 228 | dest="run_tests", 229 | default=False, 230 | help="Invoke build tests during spack install") 231 | 232 | # option to init osx sdk env flags 233 | parser.add_argument("--macos-sdk-env-setup", 234 | action="store_true", 235 | dest="macos_sdk_env_setup", 236 | default=False, 237 | help="Set several env vars to select OSX SDK settings." 238 | "This was necessary for older versions of macOS " 239 | " but can cause issues with macOS versions >= 10.13. " 240 | " so it is disabled by default.") 241 | 242 | # option to stop after spack download and setup 243 | parser.add_argument("--setup-only", 244 | action="store_true", 245 | dest="setup_only", 246 | default=False, 247 | help="Only download and setup the package manager. No further Spack command will be run. Will not create Spack Environment.") 248 | 249 | # option to stop after spack env creation 250 | parser.add_argument("--setup-and-env-only", 251 | action="store_true", 252 | dest="setup_and_env_only", 253 | default=False, 254 | help="Download and setup the package manager, create a Spack Environment. No further Spack command will be run.") 255 | 256 | # option to skip spack download and setup 257 | parser.add_argument("--skip-setup", 258 | action="store_true", 259 | dest="skip_setup", 260 | default=False, 261 | help="Only create env and install (using pre-setup Spack).") 262 | 263 | # option to skip spack download, setup and env creation 264 | parser.add_argument("--skip-setup-and-env", 265 | action="store_true", 266 | dest="skip_setup_and_env", 267 | default=False, 268 | help="Only install (using pre-setup Spack and environment).") 269 | 270 | # Spack externals list 271 | parser.add_argument("--spack-externals", 272 | dest="spack_externals", 273 | default=None, 274 | nargs="+", 275 | help="Space delimited string of packages for Spack to search for externals (if no spack_env_file is found)") 276 | 277 | # Spack compiler paths list 278 | parser.add_argument("--spack-compiler-paths", 279 | dest="spack_compiler_paths", 280 | default=None, 281 | nargs="+", 282 | help="Space delimited string of paths for Spack to search for compilers (if no spack_env_file is found)") 283 | 284 | # Spack Environment name 285 | parser.add_argument("--spack-env-name", 286 | dest="spack_env_name", 287 | default="spack_env", 288 | help="The name of the Spack Environment, which will be created in prefix directory.") 289 | 290 | # Spack Environment file 291 | parser.add_argument("--spack-env-file", 292 | dest="spack_env_file", 293 | default=None, 294 | help="Path to Spack Environment file (e.g. spack.yaml or spack.lock)") 295 | 296 | ############### 297 | # parse args 298 | ############### 299 | args, extra_args = parser.parse_known_args() 300 | # we want a dict b/c the values could 301 | # be passed without using optparse 302 | args = vars(args) 303 | 304 | # if rel path is given for the mirror, we need to evaluate here -- before any 305 | # chdirs to avoid confusion related to what it is relative to. 306 | # (it should be relative to where uberenv is run from, so it matches what you expect 307 | # from shell completion, etc) 308 | if not is_windows() and args["mirror"] is not None: 309 | if not args["mirror"].startswith(("http","oci")) and not os.path.isabs(args["mirror"]): 310 | args["mirror"] = pabs(args["mirror"]) 311 | return args, extra_args 312 | 313 | def have_internet(host="llnl.gov", port=80, timeout=3): 314 | try: 315 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 316 | s.settimeout(timeout) 317 | s.connect((host, port)) 318 | return True 319 | except: 320 | return False 321 | finally: 322 | s.close() 323 | 324 | def pretty_print_dictionary(dictionary): 325 | for key, value in dictionary.items(): 326 | print(" {0}: {1}".format(key, value)) 327 | 328 | def uberenv_script_dir(): 329 | # returns the directory of the uberenv.py script 330 | return os.path.dirname(os.path.abspath(__file__)) 331 | 332 | def load_json_file(json_file): 333 | # reads json file 334 | return json.load(open(json_file)) 335 | 336 | def is_darwin(): 337 | return "darwin" in platform.system().lower() 338 | 339 | def is_windows(): 340 | return "windows" in platform.system().lower() 341 | 342 | def find_project_config(args): 343 | project_json_file = args["project_json"] 344 | # Default case: "project.json" seats next to uberenv.py or is given on command line. 345 | if os.path.isfile(project_json_file): 346 | return project_json_file 347 | # Submodule case: Look for ".uberenv_config.json" in current then search parent dirs 348 | else: 349 | lookup_path = pabs(uberenv_script_dir()) 350 | end_of_search = False 351 | while not end_of_search: 352 | if os.path.dirname(lookup_path) == lookup_path: 353 | end_of_search = True 354 | project_json_file = pjoin(lookup_path,".uberenv_config.json") 355 | if os.path.isfile(project_json_file): 356 | return project_json_file 357 | else: 358 | lookup_path = pabs(os.path.join(lookup_path, os.pardir)) 359 | print("ERROR: No Uberenv configuration json file found") 360 | sys.exit(-1) 361 | 362 | 363 | class UberEnv(): 364 | """ Base class for package manager """ 365 | 366 | def __init__(self, args, extra_args): 367 | self.args = args 368 | self.extra_args = extra_args 369 | 370 | # load project settings 371 | self.project_args = load_json_file(args["project_json"]) 372 | 373 | # setup main package name 374 | self.pkg_name = self.set_from_args_or_json("package_name") 375 | 376 | # Set project.json defaults 377 | if not "force_commandline_prefix" in self.project_args: 378 | self.project_args["force_commandline_prefix"] = False 379 | 380 | print("[uberenv project settings: ") 381 | pretty_print_dictionary(self.project_args) 382 | print("]") 383 | 384 | print("[uberenv command line options: ") 385 | pretty_print_dictionary(self.args) 386 | print("]") 387 | 388 | def setup_paths_and_dirs(self): 389 | self.uberenv_path = uberenv_script_dir() 390 | 391 | # setup destination paths 392 | if not self.args["prefix"]: 393 | if self.project_args["force_commandline_prefix"]: 394 | # project has specified prefix must be on command line 395 | print("[ERROR: --prefix flag for library destination is required]") 396 | sys.exit(-1) 397 | # otherwise set default 398 | self.args["prefix"] = "uberenv_libs" 399 | 400 | self.dest_dir = pabs(self.args["prefix"]) 401 | 402 | # print a warning if the dest path already exists 403 | if not os.path.isdir(self.dest_dir): 404 | os.mkdir(self.dest_dir) 405 | else: 406 | print("[info: destination '{0}' already exists]".format(self.dest_dir)) 407 | 408 | def set_from_args_or_json(self,setting, optional=True): 409 | """ 410 | When optional=False: 411 | If the setting key is not in the json file, error and raise an exception. 412 | When optional=True: 413 | If the setting key is not in the json file or args, return None. 414 | """ 415 | setting_value = None 416 | try: 417 | setting_value = self.project_args[setting] 418 | except (KeyError): 419 | if not optional: 420 | print("ERROR: '{0}' must at least be defined in project.json".format(setting)) 421 | raise 422 | if self.args[setting]: 423 | setting_value = self.args[setting] 424 | return setting_value 425 | 426 | def set_from_json(self,setting, optional=True): 427 | """ 428 | When optional=False: 429 | If the setting key is not in the json file, error and raise an exception. 430 | When optional=True: 431 | If the setting key is not in the json file or args, return None. 432 | """ 433 | setting_value = None 434 | try: 435 | setting_value = self.project_args[setting] 436 | except (KeyError): 437 | if not optional: 438 | print("ERROR: '{0}' must at least be defined in project.json".format(setting)) 439 | raise 440 | return setting_value 441 | 442 | def detect_platform(self): 443 | # find supported spack.yaml 444 | if is_darwin(): 445 | return "darwin" 446 | elif "SYS_TYPE" in os.environ.keys(): 447 | return os.environ["SYS_TYPE"].lower() 448 | else: 449 | return None 450 | 451 | 452 | class VcpkgEnv(UberEnv): 453 | """ Helper to clone vcpkg and install libraries on Windows """ 454 | 455 | def __init__(self, args, extra_args): 456 | UberEnv.__init__(self,args,extra_args) 457 | 458 | # setup architecture triplet 459 | self.vcpkg_triplet = self.set_from_args_or_json("vcpkg_triplet") 460 | print("Vcpkg triplet: {}".format(self.vcpkg_triplet)) 461 | if self.vcpkg_triplet is None: 462 | self.vcpkg_triplet = os.getenv("VCPKG_DEFAULT_TRIPLET", "x86-windows") 463 | 464 | def setup_paths_and_dirs(self): 465 | # get the current working path, and the glob used to identify the 466 | # package files we want to hot-copy to vcpkg 467 | 468 | UberEnv.setup_paths_and_dirs(self) 469 | 470 | # Find path to vcpkg ports 471 | _errmsg = "" 472 | if self.args["vcpkg_ports_path"]: 473 | # Command line option case 474 | self.vcpkg_ports_path = pabs(self.args["vcpkg_ports_path"]) 475 | _errmsg = "Given path for command line option `vcpkg-ports-path` does not exist" 476 | elif "vcpkg_ports_path" in self.project_args: 477 | # .uberenv_config.json case 478 | new_path = self.project_args["vcpkg_ports_path"] 479 | if new_path is not None: 480 | self.vcpkg_ports_path = pabs(new_path) 481 | _errmsg = "Given path in config file option 'vcpkg_ports_path' does not exist" 482 | else: 483 | # next to uberenv.py script (backwards compatibility) 484 | self.vcpkg_ports_path = pabs(pjoin(self.uberenv_path, "vcpkg_ports")) 485 | _errmsg = "Could not find any directory for vcpkg ports. " \ 486 | "Use either command line option 'vcpkg-ports-path', " \ 487 | "config file option 'vcpkg_ports_path', or " \ 488 | "defaulted directory 'vcpkg_ports' next to 'uberenv.py'" 489 | 490 | if not os.path.isdir(self.vcpkg_ports_path): 491 | print("[ERROR: {0}: {1}]".format(_errmsg, self.vcpkg_ports_path)) 492 | sys.exit(-1) 493 | 494 | # setup path for vcpkg repo 495 | print("[installing to: {0}]".format(self.dest_dir)) 496 | self.dest_vcpkg = pjoin(self.dest_dir,"vcpkg") 497 | 498 | if os.path.isdir(self.dest_vcpkg): 499 | print("[info: destination '{0}' already exists]".format(self.dest_vcpkg)) 500 | 501 | def clone_repo(self): 502 | if not os.path.isdir(self.dest_vcpkg): 503 | # compose clone command for the dest path, vcpkg url and branch 504 | vcpkg_branch = self.project_args.get("vcpkg_branch", "master") 505 | vcpkg_url = self.project_args.get("vcpkg_url", "https://github.com/microsoft/vcpkg") 506 | 507 | print("[info: cloning vcpkg '{0}' branch from {1} into {2}]" 508 | .format(vcpkg_branch,vcpkg_url, self.dest_vcpkg)) 509 | 510 | os.chdir(self.dest_dir) 511 | 512 | clone_args = ("-c http.sslVerify=false " 513 | if self.args["ignore_ssl_errors"] else "") 514 | 515 | clone_cmd = "git {0} clone --single-branch -b {1} {2} vcpkg".format(clone_args, vcpkg_branch,vcpkg_url) 516 | sexe(clone_cmd, echo=True) 517 | 518 | # optionally, check out a specific commit 519 | if "vcpkg_commit" in self.project_args: 520 | sha1 = self.project_args["vcpkg_commit"] 521 | print("[info: using vcpkg commit {0}]".format(sha1)) 522 | os.chdir(self.dest_vcpkg) 523 | sexe("git checkout {0}".format(sha1),echo=True) 524 | 525 | if self.args["repo_pull"]: 526 | # do a pull to make sure we have the latest 527 | os.chdir(self.dest_vcpkg) 528 | sexe("git stash", echo=True) 529 | res = sexe("git pull", echo=True) 530 | if res != 0: 531 | #Usually untracked files that would be overwritten 532 | print("[ERROR: Git failed to pull]") 533 | sys.exit(-1) 534 | 535 | 536 | # Bootstrap vcpkg 537 | os.chdir(self.dest_vcpkg) 538 | print("[info: bootstrapping vcpkg]") 539 | sexe("bootstrap-vcpkg.bat -disableMetrics") 540 | 541 | def patch(self): 542 | """ hot-copy our ports into vcpkg """ 543 | 544 | dest_vcpkg_ports = pjoin(self.dest_vcpkg, "ports") 545 | 546 | print("[info: copying from {0} to {1}]".format(self.vcpkg_ports_path, dest_vcpkg_ports)) 547 | shutil.copytree(self.vcpkg_ports_path, dest_vcpkg_ports, dirs_exist_ok=True) 548 | 549 | 550 | def clean_build(self): 551 | pass 552 | 553 | def show_info(self): 554 | os.chdir(self.dest_vcpkg) 555 | print("[info: Details for package '{0}']".format(self.pkg_name)) 556 | sexe("vcpkg.exe search " + self.pkg_name, echo=True) 557 | 558 | print("[info: Dependencies for package '{0}']".format(self.pkg_name)) 559 | sexe("vcpkg.exe depend-info " + self.pkg_name, echo=True) 560 | 561 | def create_mirror(self): 562 | pass 563 | 564 | def use_mirror(self): 565 | pass 566 | 567 | def install(self): 568 | 569 | os.chdir(self.dest_vcpkg) 570 | install_cmd = "vcpkg.exe " 571 | install_cmd += "install {0}:{1}".format(self.pkg_name, self.vcpkg_triplet) 572 | 573 | res = sexe(install_cmd, echo=True) 574 | 575 | # Running the install_cmd eventually generates the host config file, 576 | # which we copy to the target directory. 577 | src_hc = pjoin(self.dest_vcpkg, "installed", self.vcpkg_triplet, "include", self.pkg_name, "hc.cmake") 578 | hcfg_fname = pjoin(self.dest_dir, "{0}.{1}.cmake".format(platform.uname()[1], self.vcpkg_triplet)) 579 | print("[info: copying host config file to {0}]".format(hcfg_fname)) 580 | shutil.copy(os.path.abspath(src_hc), hcfg_fname) 581 | print("") 582 | print("[install complete!]") 583 | return res 584 | 585 | 586 | class SpackEnv(UberEnv): 587 | """ Helper to clone spack and install libraries on MacOS an Linux """ 588 | 589 | def __init__(self, args, extra_args): 590 | UberEnv.__init__(self,args,extra_args) 591 | 592 | self.pkg_version = self.set_from_json("package_version") 593 | self.pkg_src_dir = self.set_from_args_or_json("package_source_dir", True) 594 | self.pkg_final_phase = self.set_from_args_or_json("package_final_phase", True) 595 | self.build_mode = self.set_from_args_or_json("spack_build_mode", True) 596 | self.spack_externals = self.set_from_args_or_json("spack_externals", True) 597 | self.spack_compiler_paths = self.set_from_args_or_json("spack_compiler_paths", True) 598 | 599 | # default spack build mode is dev-build 600 | if self.build_mode is None: 601 | self.build_mode = "dev-build" 602 | # NOTE: install always overrides the build mode to "install" 603 | if self.args["install"]: 604 | self.build_mode = "install" 605 | # if we are using fake package mode, adjust the pkg name 606 | if self.build_mode == "uberenv-pkg": 607 | self.pkg_name = "uberenv-" + self.pkg_name 608 | # convert lists to space-delimited string 609 | if type(self.spack_externals) is list: 610 | self.spack_externals = " ".join(self.spack_externals) 611 | if type(self.spack_compiler_paths) is list: 612 | self.spack_compiler_paths = " ".join(self.spack_compiler_paths) 613 | 614 | # Whether or not to generate a spack.yaml 615 | self.spack_setup_environment = False 616 | 617 | print("[uberenv spack build mode: {0}]".format(self.build_mode)) 618 | self.packages_paths = [] 619 | self.spec_hash = "" 620 | self.use_install = False 621 | 622 | # Some additional setup for macos 623 | if is_darwin(): 624 | if args["macos_sdk_env_setup"]: 625 | # setup osx deployment target and sdk settings 626 | setup_osx_sdk_env_vars() 627 | else: 628 | print("[skipping MACOSX env var setup]") 629 | 630 | # setup default spec 631 | if args["spec"] is None: 632 | if is_darwin(): 633 | # Note: newer spack, for macOS we need to use `apple-clang` 634 | args["spec"] = "%apple-clang" 635 | else: 636 | args["spec"] = "%gcc" 637 | self.args["spec"] = "@{0}{1}".format(self.pkg_version,args["spec"]) 638 | elif not args["spec"].startswith("@"): 639 | self.args["spec"] = "@{0}{1}".format(self.pkg_version,args["spec"]) 640 | else: 641 | self.args["spec"] = "{0}".format(args["spec"]) 642 | 643 | print("[spack spec: {0}]".format(self.args["spec"])) 644 | 645 | # Appends spec to package name (Example: 'magictestlib_cached@1.0.0%gcc') 646 | self.pkg_name_with_spec = "'{0}{1}'".format(self.pkg_name, self.args["spec"]) 647 | 648 | # List of concretizer options not in all versions of spack 649 | # (to be checked if it exists after cloning spack) 650 | self.fresh_exists = False 651 | self.reuse_exists = False 652 | 653 | # Spack executable (will include environment -e option by default) 654 | def spack_exe(self, use_spack_env = True): 655 | exe = pjoin(self.dest_dir, "spack/bin/spack") 656 | 657 | # Add debug flags 658 | if self.args["spack_debug"]: 659 | exe = "{0} --debug --stacktrace".format(exe) 660 | 661 | # Run Spack with environment directory 662 | if use_spack_env: 663 | exe = "{0} -D {1}".format(exe, self.spack_env_directory) 664 | 665 | return exe 666 | 667 | # Returns version of Spack being used 668 | def spack_version(self): 669 | res, out = sexe('{0} --version'.format(self.spack_exe(use_spack_env=False)), ret_output=True) 670 | return out 671 | 672 | def check_concretizer_args(self): 673 | cmd = "{0} help install".format(self.spack_exe(use_spack_env=False)) 674 | print("[Checking for concretizer options...]") 675 | res, out = sexe( cmd, ret_output = True) 676 | if "--fresh" in out: 677 | self.fresh_exists = True 678 | print("[--fresh exists.]") 679 | if "--reuse" in out: 680 | self.reuse_exists = True 681 | print("[--reuse exists.]") 682 | 683 | def add_concretizer_args(self, options): 684 | # reuse is now default in spack, if on and exists use that 685 | # otherwise use fresh if it exists 686 | if self.args["reuse"] and self.reuse_exists: 687 | options += "--reuse " 688 | elif self.fresh_exists: 689 | options += "--fresh " 690 | return options 691 | 692 | def print_spack_python_info(self): 693 | cmd = "{0} python -c \"import sys; print(sys.executable);\"".format(self.spack_exe(use_spack_env=False)) 694 | res, out = sexe( cmd, ret_output = True) 695 | print("[spack python: {0}]".format(out.strip())) 696 | 697 | def append_path_to_packages_paths(self, path, errorOnNonexistant=True): 698 | path = pabs(path) 699 | if not os.path.exists(path): 700 | if errorOnNonexistant: 701 | print("[ERROR: Given path in 'spack_packages_path' does not exist: {0}]".format(path)) 702 | sys.exit(-1) 703 | else: 704 | return 705 | self.packages_paths.append(path) 706 | 707 | 708 | def setup_paths_and_dirs(self): 709 | print("[setting up paths for environment]") 710 | # get the current working path 711 | 712 | UberEnv.setup_paths_and_dirs(self) 713 | 714 | # Next to uberenv.py (backwards compatibility) 715 | spack_configs_path = pabs(pjoin(self.uberenv_path,"spack_config")) 716 | 717 | # In project config file 718 | if "spack_configs_path" in self.project_args.keys(): 719 | new_path = self.project_args["spack_configs_path"] 720 | if new_path is not None: 721 | spack_configs_path = pabs(new_path) 722 | if not os.path.isdir(spack_configs_path): 723 | print("[ERROR: Given path in 'spack_configs_path' does not exist: {0}]".format(spack_configs_path)) 724 | sys.exit(-1) 725 | 726 | # Set spack_env_directory to absolute path and (if exists) check validity 727 | self.spack_env_name = self.args["spack_env_name"] 728 | self.spack_env_directory = pabs(os.path.join(self.dest_dir, self.spack_env_name)) 729 | if os.path.exists(self.spack_env_directory) and not self.args["skip_setup_and_env"]: 730 | print("Removing old Spack Environment Directory: {0}".format(self.spack_env_directory)) 731 | shutil.rmtree(self.spack_env_directory) 732 | 733 | # Setup path of Spack Environment file if not specified on command line 734 | # Check under spack_config_path -> detected platform -> spack.yaml/ .lock 735 | self.spack_env_file = self.args["spack_env_file"] 736 | if self.spack_env_file is None: 737 | # Check if platform is detected 738 | uberenv_plat = self.detect_platform() 739 | if not uberenv_plat is None: 740 | # Check if a path to an init file is located 741 | self.spack_env_file = pabs(pjoin(spack_configs_path, uberenv_plat)) 742 | spack_env_yaml = pjoin(self.spack_env_file, "spack.yaml") 743 | spack_env_lock = pjoin(self.spack_env_file, "spack.lock") 744 | if os.path.exists(spack_env_yaml): 745 | self.spack_env_file = spack_env_yaml 746 | elif os.path.exists(spack_env_lock): 747 | self.spack_env_file = spack_env_lock 748 | else: 749 | print("[WARNING: Could not find Spack Environment file (e.g. spack.yaml) under: {0}]".format(self.spack_env_file)) 750 | self.spack_env_file = None 751 | 752 | # Copy "defaults.yaml" and "versions.yaml" from configs dir, if they exist 753 | for _config_file in ("defaults.yaml", "versions.yaml"): 754 | _src = pjoin(spack_configs_path, _config_file) 755 | _dst = pabs(pjoin(self.spack_env_directory, "..", _config_file)) 756 | print("[checking for '{0}' yaml file]".format(_src)) 757 | if os.path.exists(_src): 758 | print("[copying '{0}' config file to {1}]".format(_config_file, _dst)) 759 | shutil.copy(_src, _dst) 760 | 761 | # If you still could not find a spack.yaml, create one later on 762 | if self.spack_env_file is None: 763 | print("[No Spack Environment file found, so Uberenv will generate one. If you do not want this behavior, then supply a Spack Environment file in // or specify one using the command line argument: --spack-env-file=/path/to/spack.yaml]") 764 | self.spack_setup_environment = True 765 | else: 766 | self.spack_env_file = pabs(self.spack_env_file) 767 | print("[Spack Environment file: {0}]".format(self.spack_env_file)) 768 | 769 | # Find project level packages to override spack's internal packages 770 | if "spack_packages_path" in self.project_args.keys(): 771 | # packages directories listed in project.json 772 | _paths = self.project_args["spack_packages_path"] 773 | if not isinstance(_paths, list): 774 | # user gave a single string 775 | self.append_path_to_packages_paths(_paths) 776 | else: 777 | # user gave a list of strings 778 | for _path in _paths: 779 | self.append_path_to_packages_paths(_path) 780 | else: 781 | # default to packages living next to uberenv script if it exists 782 | self.append_path_to_packages_paths(pjoin(self.uberenv_path,"packages"), errorOnNonexistant=False) 783 | 784 | print("[installing to: {0}]".format(self.dest_dir)) 785 | 786 | self.dest_spack = pjoin(self.dest_dir,"spack") 787 | if os.path.isdir(self.dest_spack): 788 | print("[info: destination '{0}' already exists]".format(self.dest_spack)) 789 | 790 | if self.build_mode == "dev-build": 791 | self.pkg_src_dir = os.path.abspath(os.path.join(self.uberenv_path,self.pkg_src_dir)) 792 | if not os.path.isdir(self.pkg_src_dir): 793 | print("[ERROR: package_source_dir '{0}' does not exist]".format(self.pkg_src_dir)) 794 | sys.exit(-1) 795 | 796 | def find_spack_pkg_path_from_hash(self, pkg_name, pkg_hash): 797 | res, out = sexe("{0} find -p /{1}".format(self.spack_exe(), pkg_hash), ret_output = True) 798 | for l in out.split("\n"): 799 | # TODO: at least print a warning when several choices exist. This will 800 | # pick the first in the list. 801 | if l.startswith(pkg_name) and len(l.split()) > 1: 802 | return {"name": pkg_name, "path": l.split()[-1]} 803 | print("[ERROR: Failed to find package from hash named '{0}' with hash '{1}']".format(pkg_name, pkg_hash)) 804 | sys.exit(-1) 805 | 806 | def find_spack_pkg_path(self, pkg_name, spec = ""): 807 | res, out = sexe("{0} find -p {1}".format(self.spack_exe(),self.pkg_name_with_spec), ret_output = True) 808 | for l in out.split("\n"): 809 | # TODO: at least print a warning when several choices exist. This will 810 | # pick the first in the list. 811 | if l.startswith(pkg_name) and len(l.split()) > 1: 812 | return {"name": pkg_name, "path": l.split()[-1]} 813 | print("[ERROR: Failed to find package from spec named '{0}' with spec '{1}']".format(pkg_name, spec)) 814 | sys.exit(-1) 815 | 816 | def clone_repo(self): 817 | if not os.path.isdir(self.dest_spack): 818 | 819 | # compose clone command for the dest path, spack url and branch 820 | print("[info: cloning spack develop branch from github]") 821 | 822 | os.chdir(self.dest_dir) 823 | 824 | clone_args = ("-c http.sslVerify=false " 825 | if self.args["ignore_ssl_errors"] else "") 826 | 827 | spack_url = self.project_args.get("spack_url", "https://github.com/spack/spack.git") 828 | spack_branch = self.project_args.get("spack_branch", "develop") 829 | 830 | clone_cmd = "git {0} clone --single-branch --depth=1 -b {1} {2} spack".format(clone_args, spack_branch, spack_url) 831 | sexe(clone_cmd, echo=True) 832 | 833 | if "spack_commit" in self.project_args: 834 | # optionally, check out a specific commit 835 | os.chdir(pjoin(self.dest_dir,"spack")) 836 | sha1 = self.project_args["spack_commit"] 837 | res, current_sha1 = sexe("git log -1 --pretty=%H", ret_output=True) 838 | if sha1 != current_sha1: 839 | print("[info: using spack commit {0}]".format(sha1)) 840 | sexe("git stash", echo=True) 841 | sexe("git fetch --depth=1 origin {0}".format(sha1),echo=True) 842 | res = sexe("git checkout {0}".format(sha1),echo=True) 843 | if res != 0: 844 | # Usually untracked files that would be overwritten 845 | print("[ERROR: Git failed to checkout]") 846 | sys.exit(-1) 847 | 848 | if self.args["repo_pull"]: 849 | # do a pull to make sure we have the latest 850 | os.chdir(pjoin(self.dest_dir,"spack")) 851 | sexe("git stash", echo=True) 852 | res = sexe("git pull", echo=True) 853 | if res != 0: 854 | #Usually untracked files that would be overwritten 855 | print("[ERROR: Git failed to pull]") 856 | sys.exit(-1) 857 | 858 | def disable_spack_config_scopes(self): 859 | # disables all config scopes except "defaults", which we will 860 | # force our settings into 861 | spack_lib_config = pjoin(self.dest_spack,"lib","spack","spack","config.py") 862 | print("[disabling config scope (except defaults) in: {0}]".format(spack_lib_config)) 863 | cfg_script = open(spack_lib_config).read() 864 | # 865 | # For newer versions of spack, we can use the SPACK_DISABLE_LOCAL_CONFIG 866 | # env var plumbing. We patch it to True to make a permanent change. 867 | # 868 | # Note: This path does not disable the 'site' config, but disabling 'user' config 869 | # is our primary goal. 870 | # 871 | spack_disable_env_stmt = 'disable_local_config = "SPACK_DISABLE_LOCAL_CONFIG" in os.environ' 872 | spack_disable_env_stmt_perm = "disable_local_config = True" 873 | if cfg_script.count(spack_disable_env_stmt) > 0: 874 | cfg_script = cfg_script.replace(spack_disable_env_stmt, 875 | spack_disable_env_stmt_perm) 876 | # path for older versions of spack 877 | elif cfg_script.count(spack_disable_env_stmt_perm) == 0: 878 | for cfg_scope_stmt in ["('system', os.path.join(spack.paths.system_etc_path, 'spack')),", 879 | "('site', os.path.join(spack.paths.etc_path, 'spack')),", 880 | "('user', spack.paths.user_config_path)"]: 881 | cfg_script = cfg_script.replace(cfg_scope_stmt, 882 | "#DISABLED BY UBERENV: " + cfg_scope_stmt) 883 | open(spack_lib_config,"w").write(cfg_script) 884 | 885 | def patch(self): 886 | # this is an opportunity to show spack python info post obtaining spack 887 | self.print_spack_python_info() 888 | 889 | # force spack to use only "defaults" config scope 890 | self.disable_spack_config_scopes() 891 | 892 | # setup clingo (unless specified not to) 893 | if "spack_setup_clingo" in self.project_args and self.project_args["spack_setup_clingo"] == False: 894 | print("[info: clingo will not be installed by uberenv]") 895 | else: 896 | self.setup_clingo() 897 | 898 | # Check which concretizer this version of Spack has 899 | self.check_concretizer_args() 900 | 901 | def create_spack_env(self): 902 | # Create Spack Environment 903 | print("[creating spack env]") 904 | if self.spack_env_file is None: 905 | self.spack_env_file = "" 906 | spack_create_cmd = "{0} env create -d {1} {2}".format(self.spack_exe(use_spack_env=False), 907 | self.spack_env_directory, self.spack_env_file) 908 | res = sexe(spack_create_cmd, echo=True) 909 | if res != 0: 910 | print("[ERROR: Failed to create Spack Environment]") 911 | sys.exit(-1) 912 | 913 | # Find pre-installed compilers and packages and stop uberenv.py 914 | if self.spack_setup_environment: 915 | # Finding compilers 916 | print("[finding compilers]") 917 | if self.spack_compiler_paths is None: 918 | spack_compiler_find_cmd = "{0} compiler find".format(self.spack_exe()) 919 | else: 920 | spack_compiler_find_cmd = "{0} compiler find {1}".format(self.spack_exe(), self.spack_compiler_paths) 921 | res_compiler = sexe(spack_compiler_find_cmd, echo=True) 922 | if res_compiler != 0: 923 | print("[failed to setup environment]") 924 | sys.exit(-1) 925 | 926 | # Finding externals 927 | spack_external_find_cmd = "{0} external find --not-buildable".format(self.spack_exe()) 928 | if self.spack_externals is None: 929 | print("[finding all packages Spack knows about]") 930 | spack_external_find_cmd = "{0} --all".format(spack_external_find_cmd) 931 | else: 932 | print("[finding packages from list]") 933 | spack_external_find_cmd = "{0} {1}".format(spack_external_find_cmd, self.spack_externals) 934 | res_external = sexe(spack_external_find_cmd, echo=True) 935 | if res_external != 0: 936 | print("[failed to setup environment]") 937 | sys.exit(-1) 938 | 939 | # Copy spack.yaml to where you called package source dir 940 | generated_spack_yaml = pjoin(self.spack_env_directory, "spack.yaml") 941 | copied_spack_yaml = pjoin(pabs(self.pkg_src_dir), "spack.yaml") 942 | print("[copying spack yaml file to {0}]".format(copied_spack_yaml)) 943 | sexe("cp {0} {1}".format(generated_spack_yaml, copied_spack_yaml)) 944 | 945 | print("[setup environment]") 946 | 947 | # For each package path (if there is a repo.yaml), add Spack repository to environment 948 | if len(self.packages_paths) > 0: 949 | for _base_path in self.packages_paths: 950 | spack_pkg_repo = os.path.join(_base_path, "../") 951 | spack_pkg_repo_yaml = os.path.join(_base_path, "../repo.yaml") 952 | if os.path.isfile(os.path.join(spack_pkg_repo_yaml)): 953 | print("[adding spack repo {0}]".format(spack_pkg_repo)) 954 | spack_repo_add_cmd = "{0} repo add {1}".format(self.spack_exe(), spack_pkg_repo) 955 | sexe(spack_repo_add_cmd, echo=True) 956 | else: 957 | print("[ERROR: No Spack repo.yaml detected in {0}]".format(spack_pkg_repo)) 958 | sys.exit(-1) 959 | 960 | # Add spack package 961 | print("[adding spack package]") 962 | spack_add_cmd = "{0} add {1}".format(self.spack_exe(), 963 | self.pkg_name_with_spec) 964 | sexe(spack_add_cmd, echo=True) 965 | 966 | # For dev-build, call develop 967 | if self.build_mode == "dev-build": 968 | print("[calling spack develop]") 969 | spack_develop_cmd = "{0} develop --no-clone --path={1} {2}".format( 970 | self.spack_exe(), self.pkg_src_dir, self.pkg_name_with_spec) 971 | sexe(spack_develop_cmd, echo=True) 972 | 973 | def concretize_spack_env(self): 974 | # Spack concretize 975 | print("[concretizing spack env]") 976 | spack_concretize_cmd = "{0} concretize ".format(self.spack_exe()) 977 | spack_concretize_cmd = self.add_concretizer_args(spack_concretize_cmd) 978 | sexe(spack_concretize_cmd, echo=True) 979 | 980 | def clean_build(self): 981 | # clean out any spack cached stuff (except build stages, downloads, & 982 | # spack's bootstrapping software) 983 | cln_cmd = "{0} clean --misc-cache --failures --python-cache".format(self.spack_exe(use_spack_env=False)) 984 | res = sexe(cln_cmd, echo=True) 985 | 986 | # check if we need to force uninstall of selected packages 987 | if self.args["spack_clean"]: 988 | if "spack_clean_packages" in self.project_args: 989 | for cln_pkg in self.project_args["spack_clean_packages"]: 990 | if self.find_spack_pkg_path(cln_pkg) is not None: 991 | unist_cmd = "{0} uninstall -f -y --all --dependents ".format(self.spack_exe()) + cln_pkg 992 | res = sexe(unist_cmd, echo=True) 993 | 994 | def show_info(self): 995 | # print version of spack 996 | print("[spack version: {0}]".format(self.spack_version())) 997 | 998 | # print concretized spec with install info 999 | # default case prints install status and 32 characters hash 1000 | 1001 | options = "" 1002 | options = self.add_concretizer_args(options) 1003 | options += "--install-status --very-long" 1004 | spec_cmd = "{0} spec {1}".format(self.spack_exe(), options) 1005 | 1006 | res, out = sexe(spec_cmd, ret_output=True, echo=True) 1007 | print(out) 1008 | 1009 | # Check if spec is already installed and set spec_hash 1010 | for line in out.split("\n"): 1011 | # Example of matching line: ("status" "hash" "package"...) 1012 | # [+] hf3cubkgl74ryc3qwen73kl4yfh2ijgd serac@develop%clang@10.0.0-apple~debug~devtools~glvis arch=darwin-mojave-x86_64 1013 | if re.match(r"^(\[\+\]| - ) [a-z0-9]{32} " + re.escape(self.pkg_name), line): 1014 | self.spec_hash = line.split(" ")[1].lstrip() 1015 | # if spec already installed 1016 | if line.startswith("[+]"): 1017 | pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name,self.spec_hash) 1018 | install_path = pkg_path["path"] 1019 | # testing that the path exists is mandatory until Spack team fixes 1020 | # https://github.com/spack/spack/issues/16329 1021 | if os.path.isdir(install_path): 1022 | print("[Warning: {0} has already been installed in {1}]".format(self.pkg_name_with_spec,install_path)) 1023 | print("[Warning: Uberenv will proceed using this directory]") 1024 | self.use_install = True 1025 | 1026 | return res 1027 | 1028 | 1029 | def install(self): 1030 | # use the uberenv package to trigger the right builds 1031 | # and build an host-config.cmake file 1032 | if not self.use_install: 1033 | # create install command using appropriate flags 1034 | install_cmd = self.spack_exe() + " " 1035 | 1036 | # spack flags 1037 | if self.args["ignore_ssl_errors"]: 1038 | install_cmd += "-k " 1039 | 1040 | # install flags 1041 | install_cmd += "install " 1042 | install_cmd = self.add_concretizer_args(install_cmd) 1043 | if self.build_mode == "dev-build": 1044 | install_cmd += "--keep-stage " 1045 | if self.args["spack_allow_deprecated"]: 1046 | install_cmd += "--deprecated " 1047 | if self.pkg_final_phase: 1048 | install_cmd += "-u {0} ".format(self.pkg_final_phase) 1049 | if self.args["run_tests"]: 1050 | install_cmd += "--test=root " 1051 | if self.args["build_jobs"]: 1052 | install_cmd += "-j {0} ".format(self.args["build_jobs"]) 1053 | 1054 | res = sexe(install_cmd, echo=True) 1055 | if res != 0: 1056 | print("[ERROR: Failure of spack install]") 1057 | return res 1058 | 1059 | # when using install or uberenv-pkg mode, create a symlink to the host config 1060 | if self.build_mode == "install" or \ 1061 | self.build_mode == "uberenv-pkg" \ 1062 | or self.use_install: 1063 | # only create a symlink if you're completing all phases 1064 | if self.pkg_final_phase == None or self.pkg_final_phase == "install": 1065 | # use spec_hash to locate b/c other helper won't work if complex 1066 | # deps are provided in the spec (e.g: @ver+variant ^package+variant) 1067 | pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name, self.spec_hash) 1068 | if self.pkg_name != pkg_path["name"]: 1069 | print("[ERROR: Could not find install of {0} with hash {1}]".format(self.pkg_name,self.spec_hash)) 1070 | return -1 1071 | else: 1072 | # Symlink host-config file 1073 | hc_glob = glob.glob(pjoin(pkg_path["path"],"*.cmake")) 1074 | if len(hc_glob) > 0: 1075 | hc_path = hc_glob[0] 1076 | hc_fname = os.path.split(hc_path)[1] 1077 | hc_symlink_path = pjoin(self.dest_dir,hc_fname) 1078 | if os.path.islink(hc_symlink_path): 1079 | os.unlink(hc_symlink_path) 1080 | elif os.path.isfile(hc_symlink_path): 1081 | sexe("rm -f {0}".format(hc_symlink_path)) 1082 | print("[symlinking host config file {0} to {1}]".format(hc_path,hc_symlink_path)) 1083 | os.symlink(hc_path,hc_symlink_path) 1084 | # if user opt'd for an install, we want to symlink the final 1085 | # install to an easy place: 1086 | # Symlink install directory 1087 | if self.build_mode == "install": 1088 | pkg_lnk_dir = "{0}-install".format(self.pkg_name) 1089 | if os.path.islink(pkg_lnk_dir): 1090 | os.unlink(pkg_lnk_dir) 1091 | print("") 1092 | print("[symlinking install to {0}]".format(pjoin(self.dest_dir,pkg_lnk_dir))) 1093 | os.symlink(pkg_path["path"],pabs(pkg_lnk_dir)) 1094 | print("") 1095 | print("[install complete!]") 1096 | elif self.build_mode == "dev-build": 1097 | # we are in the "only dependencies" dev build case and the host-config 1098 | # file has to be copied from the do-be-deleted spack-build dir. 1099 | build_base = pjoin(self.dest_dir,"{0}-build".format(self.pkg_name)) 1100 | build_dir = pjoin(build_base,"spack-build") 1101 | pattern = "*{0}.cmake".format(self.pkg_name) 1102 | build_dir = pjoin(self.pkg_src_dir,"spack-build") 1103 | hc_glob = glob.glob(pjoin(build_dir,pattern)) 1104 | if len(hc_glob) > 0: 1105 | hc_path = hc_glob[0] 1106 | hc_fname = os.path.split(hc_path)[1] 1107 | if os.path.islink(hc_fname): 1108 | os.unlink(hc_fname) 1109 | print("[copying host config file to {0}]".format(pjoin(self.dest_dir,hc_fname))) 1110 | sexe("cp {0} {1}".format(hc_path,hc_fname)) 1111 | print("[removing project build directory {0}]".format(pjoin(build_dir))) 1112 | sexe("rm -rf {0}".format(build_dir)) 1113 | else: 1114 | print("[ERROR: Unsupported build mode: {0}]".format(self.build_mode)) 1115 | return -1 1116 | 1117 | def get_mirror_path(self): 1118 | mirror_path = self.args["mirror"] 1119 | if not mirror_path: 1120 | print("[--create-mirror requires a mirror directory]") 1121 | sys.exit(-1) 1122 | return mirror_path 1123 | 1124 | def create_mirror(self): 1125 | """ 1126 | Creates a spack mirror for pkg_name at mirror_path. 1127 | """ 1128 | 1129 | mirror_path = self.get_mirror_path() 1130 | 1131 | mirror_cmd = "{0} ".format(self.spack_exe()) 1132 | if self.args["ignore_ssl_errors"]: 1133 | mirror_cmd += "-k " 1134 | mirror_cmd += "mirror create -d {0} --dependencies {1}".format( 1135 | mirror_path, self.pkg_name_with_spec) 1136 | return sexe(mirror_cmd, echo=True) 1137 | 1138 | def find_spack_mirror(self, mirror_name): 1139 | """ 1140 | Returns the path of a defaults scoped spack mirror with the 1141 | given name, or None if no mirror exists. 1142 | """ 1143 | res, out = sexe("{0} mirror list".format(self.spack_exe()), ret_output=True) 1144 | mirror_path = None 1145 | for mirror in out.split('\n'): 1146 | if mirror: 1147 | parts = mirror.split() 1148 | if parts[0] == mirror_name: 1149 | mirror_path = parts[1] 1150 | return mirror_path 1151 | 1152 | def use_mirror(self): 1153 | """ 1154 | Configures spack to use mirror at a given path. 1155 | """ 1156 | mirror_name = self.pkg_name 1157 | mirror_path = self.get_mirror_path() 1158 | existing_mirror_path = self.find_spack_mirror(mirror_name) 1159 | 1160 | if existing_mirror_path and mirror_path != existing_mirror_path: 1161 | # Existing mirror has different URL, error out 1162 | print("[removing existing spack mirror `{0}` @ {1}]".format(mirror_name, 1163 | existing_mirror_path)) 1164 | # 1165 | # Note: In this case, spack says it removes the mirror, but we still 1166 | # get errors when we try to add a new one, sounds like a bug 1167 | # 1168 | sexe("{0} mirror remove --scope=defaults {1} ".format(self.spack_exe(), mirror_name), 1169 | echo=True) 1170 | existing_mirror_path = None 1171 | if not existing_mirror_path: 1172 | # Add if not already there 1173 | sexe("{0} mirror add --scope=defaults {1} {2}".format( 1174 | self.spack_exe(), mirror_name, mirror_path), echo=True) 1175 | print("[using mirror {0}]".format(mirror_path)) 1176 | 1177 | def find_spack_upstream(self, upstream_name): 1178 | """ 1179 | Returns the path of a defaults scoped spack upstream with the 1180 | given name, or None if no upstream exists. 1181 | """ 1182 | upstream_path = None 1183 | 1184 | res, out = sexe('{0} config get upstreams'.format(self.spack_exe()), ret_output=True) 1185 | if (not out) and ("upstreams:" in out): 1186 | out = out.replace(' ', '') 1187 | out = out.replace('install_tree:', '') 1188 | out = out.replace(':', '') 1189 | out = out.splitlines() 1190 | out = out[1:] 1191 | upstreams = dict(zip(out[::2], out[1::2])) 1192 | 1193 | for name in upstreams.keys(): 1194 | if name == upstream_name: 1195 | upstream_path = upstreams[name] 1196 | 1197 | return upstream_path 1198 | 1199 | def use_spack_upstream(self): 1200 | """ 1201 | Configures spack to use upstream at a given path. 1202 | """ 1203 | upstream_path = self.args["upstream"] 1204 | if not upstream_path: 1205 | print("[--create-upstream requires a upstream directory]") 1206 | sys.exit(-1) 1207 | upstream_path = pabs(upstream_path) 1208 | upstream_name = self.pkg_name 1209 | existing_upstream_path = self.find_spack_upstream(upstream_name) 1210 | if (not existing_upstream_path) or (upstream_path != pabs(existing_upstream_path)): 1211 | # Existing upstream has different URL, error out 1212 | print("[removing existing spack upstream configuration file]") 1213 | sexe("rm spack/etc/spack/defaults/upstreams.yaml") 1214 | with open('spack/etc/spack/defaults/upstreams.yaml','w+') as upstreams_cfg_file: 1215 | upstreams_cfg_file.write("upstreams:\n") 1216 | upstreams_cfg_file.write(" {0}:\n".format(upstream_name)) 1217 | upstreams_cfg_file.write(" install_tree: {0}\n".format(upstream_path)) 1218 | 1219 | def setup_clingo(self): 1220 | """ 1221 | Attempts to install the clingo answer set programming library via Spack 1222 | if it is not already available as a Python module 1223 | """ 1224 | if not have_internet(): 1225 | print("[WARNING: No internet detected. Skipping setting up clingo.]") 1226 | return 1227 | 1228 | res = sexe('{0} bootstrap now'.format(self.spack_exe(use_spack_env = False)), echo=True) 1229 | if res != 0: 1230 | print("[ERROR: 'spack bootstrap now' failed with returncode {0}]".format(res)) 1231 | sys.exit(-1) 1232 | 1233 | res = sexe('{0} bootstrap status'.format(self.spack_exe(use_spack_env = False)), echo=True) 1234 | if res != 0: 1235 | print("[ERROR: 'spack bootstrap status' failed with returncode {0}]".format(res)) 1236 | sys.exit(-1) 1237 | 1238 | 1239 | def find_osx_sdks(): 1240 | """ 1241 | Finds installed osx sdks, returns dict mapping version to file system path 1242 | """ 1243 | res = {} 1244 | sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") 1245 | for sdk in sdks: 1246 | sdk_base = os.path.split(sdk)[1] 1247 | ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] 1248 | res[ver] = sdk 1249 | return res 1250 | 1251 | def setup_osx_sdk_env_vars(): 1252 | """ 1253 | Finds installed osx sdks, returns dict mapping version to file system path 1254 | """ 1255 | # find current osx version (10.11.6) 1256 | dep_tgt = platform.mac_ver()[0] 1257 | # sdk file names use short version (ex: 10.11) 1258 | dep_tgt_short = dep_tgt[:dep_tgt.rfind(".")] 1259 | # find installed sdks, ideally we want the sdk that matches the current os 1260 | sdk_root = None 1261 | sdks = find_osx_sdks() 1262 | if dep_tgt_short in sdks.keys(): 1263 | # matches our osx, use this one 1264 | sdk_root = sdks[dep_tgt_short] 1265 | elif len(sdks) > 0: 1266 | # for now, choose first one: 1267 | dep_tgt = sdks.keys()[0] 1268 | sdk_root = sdks[dep_tgt] 1269 | else: 1270 | # no valid sdks, error out 1271 | print("[ERROR: Could not find OSX SDK @ /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/]") 1272 | sys.exit(-1) 1273 | 1274 | env["MACOSX_DEPLOYMENT_TARGET"] = dep_tgt 1275 | env["SDKROOT"] = sdk_root 1276 | print("[setting MACOSX_DEPLOYMENT_TARGET to {0}]".format(env["MACOSX_DEPLOYMENT_TARGET"])) 1277 | print("[setting SDKROOT to {0}]".format(env[ "SDKROOT"])) 1278 | 1279 | 1280 | def print_uberenv_python_info(): 1281 | print("[uberenv python: {0}]".format(sys.executable)) 1282 | 1283 | 1284 | def main(): 1285 | """ 1286 | Clones and runs a package manager to setup third_party libs. 1287 | Also creates a host-config.cmake file that can be used by our project. 1288 | """ 1289 | 1290 | print_uberenv_python_info() 1291 | 1292 | # parse args from command line 1293 | args, extra_args = parse_args() 1294 | 1295 | # project options 1296 | args["project_json"] = find_project_config(args) 1297 | 1298 | # Initialize the environment -- use vcpkg on windows, spack otherwise 1299 | env = SpackEnv(args, extra_args) if not is_windows() else VcpkgEnv(args, extra_args) 1300 | 1301 | # Setup the necessary paths and directories 1302 | env.setup_paths_and_dirs() 1303 | 1304 | # Go to package manager's destination 1305 | os.chdir(env.dest_dir) 1306 | 1307 | # Setup package manager 1308 | if not args["skip_setup"] and not args["skip_setup_and_env"]: 1309 | # Clone the package manager 1310 | env.clone_repo() 1311 | 1312 | # Patch the package manager, as necessary 1313 | env.patch() 1314 | 1315 | # Clean the build 1316 | env.clean_build() 1317 | 1318 | # Allow to end uberenv after Spack is ready 1319 | if args["setup_only"]: 1320 | 1321 | # Use Spack upstream 1322 | if not is_windows() and args["upstream"] is not None: 1323 | env.use_spack_upstream() 1324 | 1325 | return 0 1326 | 1327 | # Create Spack Environment and setup Spack package repos 1328 | if not is_windows() and not args["skip_setup_and_env"]: 1329 | env.create_spack_env() 1330 | 1331 | # Allow to end uberenv after Spack environment is ready 1332 | if args["setup_and_env_only"]: 1333 | return 0 1334 | 1335 | ########################################################### 1336 | # We now have an instance of our package manager configured, 1337 | # now we need it to build our TPLs. At this point, there are 1338 | # two possible next steps: 1339 | # 1340 | # *) create a mirror of the packages 1341 | # OR 1342 | # *) build 1343 | # 1344 | ########################################################### 1345 | if args["create_mirror"]: 1346 | return env.create_mirror() 1347 | else: 1348 | # Add mirror 1349 | if args["mirror"] is not None: 1350 | env.use_mirror() 1351 | 1352 | # Use Spack upstream 1353 | if not is_windows() and args["upstream"] is not None: 1354 | env.use_spack_upstream() 1355 | 1356 | # Concretize the spack environment 1357 | if not is_windows(): 1358 | env.concretize_spack_env() 1359 | 1360 | # Show the spec for what will be built 1361 | env.show_info() 1362 | 1363 | # Install 1364 | return env.install() 1365 | 1366 | if __name__ == "__main__": 1367 | sys.exit(main()) 1368 | --------------------------------------------------------------------------------