├── .gitignore ├── .sregistry ├── README.md ├── build.py ├── fwk394.json ├── push.py └── recipe.py ├── LICENSE ├── README.md ├── deprecated ├── README.md ├── cling-test │ ├── README.md │ ├── clingTest-xeus-cling-cuda │ └── run_shell.sh ├── config_generator.sh ├── dev-xeus-cling-cuda ├── jupyter_kernels │ ├── kernels.tar │ ├── xeus-cling-cpp11-cuda │ │ └── kernel.json.in │ ├── xeus-cling-cpp14-cuda │ │ └── kernel.json.in │ └── xeus-cling-cpp17-cuda │ │ └── kernel.json.in └── kdevelop │ ├── README.md │ ├── config_generator.sh │ ├── kdev-xeus-cling-cuda │ ├── run_kdevelop.sh │ └── run_shell.sh ├── dev_container.py ├── rel_container.py └── xcc ├── __init__.py ├── basestage.py ├── cling.py ├── config.py ├── generator.py ├── helper.py ├── jupyter.py ├── miniconda.py ├── openssl.py └── xeuscling.py /.gitignore: -------------------------------------------------------------------------------- 1 | # files and folders created by default values 2 | builds/ 3 | deprecated/config.json 4 | *.sing 5 | *.img 6 | *.sif 7 | *.def 8 | deprecated/jupyter_kernels/xeus-cling-cpp11-cuda/kernel.json 9 | deprecated/jupyter_kernels/xeus-cling-cpp14-cuda/kernel.json 10 | deprecated/jupyter_kernels/xeus-cling-cpp17-cuda/kernel.json 11 | __pycache__/ 12 | .mypy_cache/ 13 | 14 | # files and folder generated by jupyter notebook 15 | .ipynb_checkpoints/ 16 | *.ipynb 17 | 18 | # files and folders created by using the kdevelop container 19 | kdevelop/kdev_config.json 20 | kdevelop/kdev-xeus-cling-cuda.sing 21 | 22 | # files generated by the sregistry push process 23 | .sregistry/build.log 24 | .sregistry/build_libcxx.log 25 | -------------------------------------------------------------------------------- /.sregistry/README.md: -------------------------------------------------------------------------------- 1 | # Before the first push 2 | 3 | Before you can push the image to the sregistry the first time, you have to setup some keys for signing images and logging in. 4 | 5 | 1. setup key with `singularity key newpair` 6 | 2. generate a key API at https://cloud.sylabs.io/keystore (30 days valid?) and store it at `~/.singularity/sylabs-token` 7 | 8 | # Build and push 9 | 10 | ```bash 11 | # create the recipes and build the containers 12 | python build.py 13 | # push the containers to the registry 14 | python push.py 15 | ``` 16 | 17 | Use `python recipe.py ` to create only a container recipe without building the container. 18 | -------------------------------------------------------------------------------- /.sregistry/build.py: -------------------------------------------------------------------------------- 1 | import json, sys, os 2 | import shutil, subprocess 3 | from typing import Dict 4 | import recipe as rc 5 | 6 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 7 | import xcc.generator as gn 8 | 9 | def main(): 10 | if len(sys.argv) < 2: 11 | print('usage: python3 build.py /path/config.json') 12 | exit(0) 13 | 14 | check_singularity() 15 | 16 | # load the number of compile and link threads from a json file 17 | with open(str(sys.argv[1])) as config_file: 18 | config = json.load(config_file)['build'] 19 | config_file.close() 20 | 21 | print('compile threads: ' + str(config['compile_threads']) + '\n' 22 | 'linker threads: ' + str(config['linker_threads']) + '\n') 23 | 24 | answer = '' 25 | while answer not in ('y', 'n'): 26 | answer = input('is the config correct? [y/n] : ') 27 | if answer == 'y': 28 | rc.create(config, False) 29 | build(False) 30 | rc.create(config, True) 31 | build(True) 32 | if answer == 'n': 33 | exit(1) 34 | 35 | def check_singularity(): 36 | """Check if the singularity container software is available and runs 'singularity --version' 37 | 38 | """ 39 | if not shutil.which('singularity'): 40 | print('could not find singularity') 41 | exit(1) 42 | 43 | process = subprocess.Popen(['singularity', '--version'], stdout=subprocess.PIPE) 44 | output, error = process.communicate() 45 | if error is not None: 46 | print('could not run "singularity --version"') 47 | exit(1) 48 | 49 | print(output.decode("utf-8")) 50 | 51 | def build(libcxx : bool): 52 | """Generate the singularity recipe and build it. 53 | :param libcxx: build the container with libc++ 54 | :type libcxx: bool 55 | 56 | """ 57 | if libcxx: 58 | recipe_name = 'recipe_libcxx.def' 59 | image_name = 'xeus-cling-cuda-container-cxx.sif' 60 | log_name = 'build_libcxx.log' 61 | else: 62 | recipe_name = 'recipe.def' 63 | image_name = 'xeus-cling-cuda-container.sif' 64 | log_name = 'build.log' 65 | 66 | # build image 67 | process = subprocess.Popen(['singularity', 68 | 'build', 69 | '--fakeroot', 70 | image_name, 71 | recipe_name], 72 | stdout=subprocess.PIPE) 73 | output, error = process.communicate() 74 | if error is not None: 75 | print('"singularity build --fakeroot ' + image_name + ' ' + recipe_name + '" failed') 76 | exit(1) 77 | 78 | with open(log_name, 'w') as build_log: 79 | build_log.write(output.decode('utf-8')) 80 | build_log.close() 81 | 82 | if __name__ == '__main__': 83 | main() 84 | -------------------------------------------------------------------------------- /.sregistry/fwk394.json: -------------------------------------------------------------------------------- 1 | { 2 | "build" : { 3 | "compile_threads" : 28, 4 | "linker_threads" : 14 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /.sregistry/push.py: -------------------------------------------------------------------------------- 1 | import sys, json, os 2 | import shutil, subprocess 3 | 4 | def main(): 5 | check_singularity() 6 | 7 | # extract the version from the container 8 | container_version = get_container_version() 9 | if container_version is None: 10 | print('could not find container version in xeus-cling-cuda-container') 11 | exit(1) 12 | 13 | answer = '' 14 | while answer not in ('y', 'n'): 15 | answer = input('is version ' + container_version + ' correct? [y/n] : ') 16 | if answer == 'y': 17 | login_sregistry() 18 | sign_container(False) 19 | sign_container(True) 20 | push_container(container_version, False) 21 | push_container('latest', False) 22 | push_container(container_version, True) 23 | push_container('latest', True) 24 | if answer == 'n': 25 | exit(1) 26 | 27 | def check_singularity(): 28 | """Check if the singularity container software is available and runs 'singularity --version' 29 | 30 | """ 31 | if not shutil.which('singularity'): 32 | print('could not find singularity') 33 | exit(1) 34 | 35 | process = subprocess.Popen(['singularity', '--version'], stdout=subprocess.PIPE) 36 | output, error = process.communicate() 37 | if error is not None: 38 | print('could not run "singularity --version"') 39 | exit(1) 40 | 41 | print(output.decode("utf-8")) 42 | 43 | def get_container_version(): 44 | """Extract the version from the xeus-cling-cuda-container 45 | 46 | """ 47 | c_version = 'singularity inspect -d xeus-cling-cuda-container.sif' 48 | p_version = subprocess.Popen(c_version.split(), stdout=subprocess.PIPE) 49 | output, error = p_version.communicate() 50 | 51 | if error is not None: 52 | print('could not run: ' + c_version) 53 | exit(1) 54 | 55 | for line in output.decode("utf-8").split('\n'): 56 | if 'XCC Version' in line: 57 | # since the tag is generated, it should have same form every time 58 | # XCC Version x.x 59 | return line.strip()[len('XCC Version '):] 60 | 61 | return None 62 | 63 | def login_sregistry(): 64 | """Login to the sregistry account 65 | 66 | """ 67 | # login to the remote server 68 | c_login = 'singularity remote login --tokenfile ' + str(os.getenv("HOME")) + '/.singularity/sylabs-token' 69 | p_login = subprocess.Popen(c_login.split()) 70 | output, error = p_login.communicate() 71 | 72 | if error is not None: 73 | print('could not run: ' + c_login) 74 | exit(1) 75 | 76 | 77 | def sign_container(libcxx : bool): 78 | """Sign the container with the private key 79 | 80 | :param libcxx: sign the libc++ container version 81 | :type libcxx: bool 82 | 83 | """ 84 | image_name = 'xeus-cling-cuda-container-cxx.sif' if libcxx else 'xeus-cling-cuda-container.sif' 85 | c_sign = 'singularity sign ' + image_name 86 | p_sign = subprocess.Popen(c_sign.split()) 87 | output, error = p_sign.communicate() 88 | 89 | if error is not None: 90 | print('could not run: ' + c_sign) 91 | exit(1) 92 | 93 | 94 | def push_container(version : str, libcxx : bool): 95 | """upload the image 96 | 97 | :param version: is used for the tag 98 | :type version: str 99 | :param libcxx: push the libc++ container version 100 | :type libcxx: bool 101 | 102 | """ 103 | image_name = 'xeus-cling-cuda-container-cxx.sif' if libcxx else 'xeus-cling-cuda-container.sif' 104 | library_name = 'xeus-cling-cuda-cxx:' if libcxx else 'xeus-cling-cuda:' 105 | 106 | # upload the container 107 | c_push = 'singularity push ' + image_name + ' library://sehrig/default/' + library_name + version 108 | p_push = subprocess.Popen(c_push.split()) 109 | output, error = p_push.communicate() 110 | 111 | if error is not None: 112 | print('could not run: ' + c_push) 113 | exit(1) 114 | 115 | if __name__ == '__main__': 116 | main() 117 | -------------------------------------------------------------------------------- /.sregistry/recipe.py: -------------------------------------------------------------------------------- 1 | import json, sys, os 2 | import shutil, subprocess 3 | from typing import Dict 4 | 5 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 6 | import xcc.generator as gn 7 | 8 | 9 | def main(): 10 | if len(sys.argv) < 2: 11 | print('usage: python3 recipe.py /path/config.json') 12 | exit(0) 13 | with open(str(sys.argv[1])) as config_file: 14 | config = json.load(config_file)['build'] 15 | config_file.close() 16 | 17 | print('compile threads: ' + str(config['compile_threads']) + '\n' 18 | 'linker threads: ' + str(config['linker_threads'])) 19 | 20 | create(config, True) 21 | create(config, False) 22 | 23 | def create(config : Dict, libcxx : bool): 24 | """Generate the singularity recipe. 25 | 26 | :param config: Json config with number of compile and linker threads 27 | :type config: Dict 28 | :param libcxx: build the container with libc++ 29 | :type libcxx: bool 30 | 31 | """ 32 | if libcxx: 33 | recipe_name = 'recipe_libcxx.def' 34 | image_name = 'xeus-cling-cuda-container-cxx.sif' 35 | log_name = 'build_libcxx.log' 36 | else: 37 | recipe_name = 'recipe.def' 38 | image_name = 'xeus-cling-cuda-container.sif' 39 | log_name = 'build.log' 40 | 41 | # generate recipe 42 | xcc_gen = gn.XCC_gen(build_prefix='/opt', 43 | threads=config['compile_threads'], 44 | linker_threads=config['linker_threads'], 45 | build_libcxx=libcxx) 46 | with open(recipe_name, 'w') as recipe_file: 47 | recipe_file.write(xcc_gen.gen_release_single_stage().__str__()) 48 | recipe_file.close() 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Xeus-Cling-Cuda-Container 2 | The repository contains container recipes to build the entire stack of Xeus-Cling and Cling including cuda extension with just a few commands. 3 | 4 | # General Information 5 | The containers contain Xeus-Cling, Jupyter Notebook and Lab and the latest version of Xeus-Cling-CUDA (https://github.com/SimeonEhrig/cling/tree/test_release). 6 | 7 | Various containers are available. All recipes are generated using Python scripts and the [hpccm](https://github.com/NVIDIA/hpc-container-maker) library. 8 | 9 | - release 10 | - it can generate recipes for 11 | - singularity (official supported) 12 | - docker (experimental) 13 | - contains a fully integrated xeus-cling-cuda stack built in release mode 14 | - singularity need extra flag `--no-home` for full isolation 15 | - dev 16 | - partly integrated into the container 17 | - all libraries which should not to changed are integrated in the container 18 | - miniconda, cling and xeus-cling are built outside the container and allow modifications on the code 19 | 20 | It is also possible to [install](https://github.com/QuantStack/xeus-cling) xeus-cling via conda. But this installation use cling version 0.5 and does not support CUDA. 21 | 22 | # General Requirements 23 | 24 | To build and use the container, your host system needs two prerequisites: 25 | - [Singularity](http://singularity.lbl.gov/) >= 3.3.0 26 | - Nvidia CUDA Driver, which supports CUDA >= 8.0 27 | 28 | # Building Containers 29 | 30 | ## General hints 31 | 32 | * **Hint 1:** Cling requires a lot of RAM to build. Be careful when setting the number of threads. Otherwise you will get an out-of-memory error at link time. Working memory thread combinations are: 33 | * 4 Threads with 32 GB RAM 34 | * 14 Threads with 128 GB RAM 35 | * **Hint 2:** Be careful with hyperthreading. It can drastically change the memory usage. 36 | * **Hint 3:** If you use Singularity and do not have root permission on your system, you can use the argument `--fakeroot` or you can build the container on another system with root permission and copy it to your target system. 37 | 38 | ## Release 39 | The recipes are written in Python with [hpccm](https://github.com/NVIDIA/hpc-container-maker). No container images are created directly. Instead it creates recipes for singularity and docker. To build a singularity container, follow these steps. 40 | 41 | ```bash 42 | # create recipe 43 | python rel_container.py -o rel-xeus-cling-cuda.def 44 | # build container 45 | singularity build --fakeroot rel-xeus-cling-cuda.sif rel-xeus-cling-cuda.def 46 | ``` 47 | 48 | Use the `python rel-container.py --help` command to display all possible recipe configuration options. For example, you can set the number of threads with `python rel-container.py -j 4 -o rel-xeus-cling-cuda` (by default, all threads of the system are used). 49 | 50 | ## Dev 51 | 52 | The development container is also generated via Python script and built via Singularity. In addition to the normal build process, there is a second build stage. In this step, the source code of the projects to be further developed is downloaded and built. This is necessary because the container is read-only. The files of this step are stored on the host system, e.g. a folder in the home directory. 53 | 54 | ```bash 55 | # create recipe 56 | python dev_container.py -o dev-xeus-cling-cuda.def --project_path=/home/user/project/cling 57 | # build container 58 | singularity build --fakeroot dev-xeus-cling-cuda.sif dev-xeus-cling-cuda.def 59 | singularity run dev-xeus-cling-cuda.sif 60 | ``` 61 | 62 | * **Hint 1:** Relative `project_path`s are automatically converted to absolute paths. 63 | * **Hint 2:** Depending on the `XCC_BUILD_TYPE` the build may require a lot of storage space. The `Debug` build needs about 82 GB. 64 | 65 | Use the `python dev-container.py --help` command to display all possible recipe configuration options. 66 | 67 | # Downloading Container from the Registry 68 | 69 | The built release containers are also available in the singularity register: 70 | 71 | ```bash 72 | singularity pull library://sehrig/default/xeus-cling-cuda 73 | ``` 74 | 75 | or 76 | 77 | ```bash 78 | # the stack was built with the LLVM's libc++ and cling used libc++ (solves some problems) 79 | singularity pull library://sehrig/default/xeus-cling-cuda-cxx 80 | ``` 81 | 82 | # Running 83 | To use the xeus-cling-cuda stack via jupyter notebook use the following command. 84 | 85 | ``` bash 86 | singularity exec --nv rel-xeus-cling-cuda.sif jupyter-notebook 87 | ``` 88 | 89 | To start jupyter-lab, you must use the following command. 90 | 91 | ``` bash 92 | singularity exec --nv -B /run/user/$(id -u):/run/user/$(id -u) rel-xeus-cling-cuda.sif jupyter-lab 93 | ``` 94 | 95 | * **Hint 1:** If you get a CUDA driver error at runtime, you may have forgotten to set the `--nv` flag. 96 | * **Hint 2:** If you are using a SSH connection, do not forget the [port forwarding](https://help.ubuntu.com/community/SSH/OpenSSH/PortForwarding) for port 8888. 97 | * **Hint 3:** If you want fully isolation (e.g. because you have problems with other kernel configurations in your home directory) use the `--no-home` argument and manually bind a directory for notebooks via `-B /path/on/host:/path/in/container/`. 98 | 99 | # Development 100 | 101 | If you change the code of xeus-cling or cling, you need to rebuild the applications. There are two ways to rebuild the application. 102 | 103 | ## via interface shell 104 | It runs an interactive shell session 105 | 106 | ``` bash 107 | singularity shell --nv dev-xeus-cling-cuda.sif 108 | ``` 109 | 110 | ## via exec command 111 | Run the container, executes the commands inside the container and exit it. 112 | 113 | ``` bash 114 | singularity exec --nv dev-xeus-cling-cuda.sif cd path/to/code && make 115 | ``` 116 | -------------------------------------------------------------------------------- /deprecated/README.md: -------------------------------------------------------------------------------- 1 | # Xeus-Cling-Cuda-Container 2 | The repository contains container recipes to build the entire stack of Xeus-Cling and Cling including cuda extension with just a few commands. 3 | 4 | # General Information 5 | The containers contain Xeus-Cling, Jupyter Notebook and Lab and the latest version of Xeus-Cling-CUDA (https://github.com/SimeonEhrig/cling/tree/test_release). 6 | 7 | Various containers are available 8 | 9 | - release 10 | - recipe is written in Python with [hpccm](https://github.com/NVIDIA/hpc-container-maker) 11 | - it can generate recipes for 12 | - singularity (official supported) 13 | - docker (experimental) 14 | - contains a fully integrated xeus-cling-cuda stack built in release mode 15 | - singularity need extra flag `--no-home` for full isolation 16 | - dev 17 | - partly integrated into the container 18 | - all libraries which should not to changed are integrated in the container 19 | - miniconda, cling and xeus-cling are built outside the container and allow modifications on the code 20 | 21 | It is also possible to [install](https://github.com/QuantStack/xeus-cling) xeus-cling via conda. But this installation use cling version 0.5 and does not support CUDA. 22 | 23 | # Installation 24 | 25 | To build and use the container, your host system needs two prerequisites: 26 | - [Singularity](http://singularity.lbl.gov/) >= 2.6.0 27 | - Nvidia CUDA Driver, which supports CUDA >= 8.0 28 | 29 | ## General hints 30 | 31 | * **Hint 1:** Cling requires a lot of RAM to build. Be careful when setting the number of threads. Otherwise you will get an out-of-memory error at link time. Working memory thread combinations are: 32 | * 4 Threads with 32 GB RAM 33 | * 14 Threads with 128 GB RAM 34 | 35 | * **Hint 2:** Be careful with hyperthreading. It can drastically change the memory usage. 36 | 37 | * **Hint 3:** If you do not have root permission on your system, you can build the container on another system with root permission and copy it to your target system. 38 | 39 | ## Release 40 | The recipes are written in Python with [hpccm](https://github.com/NVIDIA/hpc-container-maker). No container images are created directly. Instead it creates recipes for singularity and docker. To build a singularity container, follow these steps. 41 | 42 | ```bash 43 | # create recipe 44 | python rel-container.py -o rel-xeus-cling-cuda 45 | # build container 46 | sudo singularity build rel-xeus-cling-cuda.sing rel-xeus-cling-cuda 47 | ``` 48 | 49 | Use the `python rel-container.py --help` command to display all possible recipe configuration options. For example, you can set the number of threads with `python rel-container.py -j 4 -o rel-xeus-cling-cuda` (by default, all threads of the system are used). 50 | 51 | ## Dev 52 | The installation of the dev container is divided into three parts, because the modification of the source code of cling and xeus-cling by the user is possible. The three parts are: 53 | - generating a configuration 54 | - build container as root 55 | - build applications as user 56 | 57 | ### Configuration 58 | You can write the `config.json` yourself or use the `generate_config.sh` script to generate it. The `config.json` must be in same folder where you start the container build process. 59 | 60 | Structure of the `config.json` 61 | ```json 62 | { 63 | "XCC_PROJECT_PATH" : "/path/to/miniconda_xeus-cling_cling_installation", 64 | "XCC_BUILD_TYPE" : "CMAKE_BUILD_TYPE", 65 | "XCC_NUM_THREADS" : "Number or empty like make -j" 66 | } 67 | ``` 68 | 69 | ### Build Container as Root 70 | 71 | To build the container, simply use the following command in the project folder. 72 | 73 | ``` bash 74 | sudo singularity build dev-xeus-cling-cuda.sing dev-xeus-cling-cuda 75 | ``` 76 | 77 | 78 | ### Build Applications as User 79 | 80 | **Attention**, this step cannot be performed on another system. It must run on the target system. 81 | 82 | This step builds miniconda, cling and xeus-cling with your user permission in the path `$XCC_PROJECT_PATH`. To start the build process, simply use the following command. 83 | 84 | ``` bash 85 | singularity run dev-xeus-cling-cuda.sing 86 | ``` 87 | 88 | **Hint:** Depending on the `XCC_BUILD_TYPE` the build may require a lot of storage space. The `Debug` build needs about 82 GB. 89 | 90 | # Running 91 | To use the xeus-cling-cuda stack via jupyter notebook use the following command. 92 | 93 | ``` bash 94 | singularity exec --nv rel-xeus-cling-cuda.sing jupyter-notebook 95 | ``` 96 | 97 | To start jupyter-lab, you must use the following command. 98 | 99 | ``` bash 100 | singularity exec --nv -B /run/user/$(id -u):/run/user/$(id -u) rel-xeus-cling-cuda.sing jupyter-lab 101 | ``` 102 | 103 | * **Hint 1:** If you get a CUDA driver error at runtime, you may have forgotten to set the `--nv` flag. 104 | * **Hint 2:** If you are using a SSH connection, do not forget the [port forwarding](https://help.ubuntu.com/community/SSH/OpenSSH/PortForwarding) for port 8888. 105 | * **Hint 3:** If you want fully isolation (e.g. because you have problems with other kernel configurations in your home directory) use the `--no-home` argument and manually bind a directory for notebooks via `-B /path/on/host:/path/in/container/`. 106 | 107 | # Development 108 | 109 | If you change the code of xeus-cling or cling, you need to rebuild the applications. There are two commands to run `make` and `make install` for cling or xeus-cling. 110 | 111 | ``` bash 112 | singularity run --app build_cling dev-xeus-cling-cuda.sing 113 | ``` 114 | 115 | ``` bash 116 | singularity run --app build_xeus-cling dev-xeus-cling-cuda.sing 117 | ``` 118 | -------------------------------------------------------------------------------- /deprecated/cling-test/README.md: -------------------------------------------------------------------------------- 1 | # Test environment extension for `make cling-test` 2 | 3 | This container extends the dev-xeus-cling-cuda.sing container by Python 2.7 to run `make cling-test`. The extension is completely optional. 4 | 5 | # About 6 | Python 2.7 is installed in an extra container because it may cause some problems with the jupyter kernel. 7 | 8 | # Installation 9 | 10 | At first, you have to [build](../README.md) the `dev-xeus-cling-cuda.sing` container. Then run the following command. 11 | ```bash 12 | sudo singularity build clingTest-xeus-cling-cuda.sing clingTest-xeus-cling-cuda 13 | ``` 14 | 15 | Then you can run the new container with the script `./run_shell`. 16 | 17 | # Usage 18 | 19 | **Import:** The Python path of the cling-test tool is set at build time. Therefore, you must create a new cmake build with the `cmake` command. Alternatively, you can run again the `cmake` on a existing build. If the `cmake` prints the following line, everything is fine. 20 | 21 | ```bash 22 | ... 23 | -- Found PythonInterp: /usr/bin/python2.7 (found version "2.7.12") 24 | ... 25 | ``` 26 | -------------------------------------------------------------------------------- /deprecated/cling-test/clingTest-xeus-cling-cuda: -------------------------------------------------------------------------------- 1 | Bootstrap: localimage 2 | From: ../dev-xeus-cling-cuda.sing 3 | 4 | %help 5 | The container extends the dev-xeus-cling-cuda container by Python 2.7 to run 'make cling-test'. 6 | The test environment is in an additional container because Python 2.7 may cause some problems with the Jupyter kernel. 7 | 8 | %setup 9 | 10 | %files 11 | 12 | %labels 13 | Maintainer Simeon Ehrig 14 | Email s.ehrig@hzdr.de 15 | Version 1.0 16 | 17 | %environment 18 | 19 | %post 20 | # This file contains all custom installations, which was installed during the post phase of building the container. It avoids errors caused by double installation, if the container is build again with a changed configuration. 21 | if [ ! -f /opt/installed_tools.txt ]; then 22 | touch /opt/installed_tools.txt 23 | fi 24 | 25 | ############################################################################################ 26 | ### install software from package manager 27 | ############################################################################################ 28 | apt update 29 | apt install -y python2.7 30 | 31 | %runscript 32 | -------------------------------------------------------------------------------- /deprecated/cling-test/run_shell.sh: -------------------------------------------------------------------------------- 1 | singularity shell --nv clingTest-xeus-cling-cuda.sing 2 | -------------------------------------------------------------------------------- /deprecated/config_generator.sh: -------------------------------------------------------------------------------- 1 | echo -n "project path (default: ./builds): " 2 | read XCC_PROJECT_PATH 3 | if [ "$XCC_PROJECT_PATH" == "" ]; 4 | then 5 | XCC_PROJECT_PATH=$PWD/builds 6 | fi 7 | 8 | while true; 9 | do 10 | echo -n "build type [Debug, Release, RelWithDebInfo, MinSizeRel]: " 11 | read XCC_BUILD_TYPE 12 | case $XCC_BUILD_TYPE in 13 | [Debug]* ) break;; 14 | [Release]* ) break;; 15 | [RelWithDebInfo]* ) break;; 16 | [MinSizeRel]* ) break;; 17 | esac 18 | done 19 | 20 | while true ; 21 | do 22 | echo -n "enter number of threads for build or leave it free to use all threads: " 23 | read XCC_NUM_THREADS 24 | if [[ $XCC_NUM_THREADS =~ ^[0-9]+$ ]] || [ -z "$XCC_NUM_THREADS" ]; 25 | then 26 | break 27 | fi 28 | done 29 | 30 | 31 | echo "{" > config.json 32 | echo -e "\t\"XCC_PROJECT_PATH\" : \"$XCC_PROJECT_PATH\"," >> config.json 33 | echo -e "\t\"XCC_BUILD_TYPE\" : \"$XCC_BUILD_TYPE\"," >> config.json 34 | echo -e "\t\"XCC_NUM_THREADS\" : \"$XCC_NUM_THREADS\"" >> config.json 35 | echo "}" >> config.json 36 | -------------------------------------------------------------------------------- /deprecated/dev-xeus-cling-cuda: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: nvidia/cuda:8.0-devel-ubuntu16.04 3 | 4 | %help 5 | This container contains a Ubuntu 16.04 environment with a CUDA 8 SDK. It use a docker container developed by Nvidia as template: https://hub.docker.com/r/nvidia/cuda/ 6 | 7 | - installed via apt 8 | - nano 9 | - python 3 10 | - wget 11 | - git 12 | - pkg-config 13 | - uuid-dev 14 | - gdb 15 | - localse-all 16 | - installed applications 17 | - cmake 3.14 (binary build via tar.gz) 18 | - installed libraries 19 | - libzmq v4.2.5 20 | - cppzmq v4.3.0 21 | - nlohmann JSON v3.3.0 22 | - xtl 0.6.5 23 | - libssl-dev 1.1.1c 24 | - xeus 0.20.0 25 | - pugixml v1.8.1 26 | - cxxopts v2.1.1 27 | - applications in a external folder 28 | - Miniconda 3 29 | - xeus-cling 0.6.0 30 | - cling 0.6~dev 31 | 32 | %setup 33 | 34 | %files 35 | config.json /opt/config.json 36 | 37 | %labels 38 | Maintainer Simeon Ehrig 39 | Email s.ehrig@hzdr.de 40 | Version 1.1 41 | 42 | %environment 43 | export PATH=$PATH:/usr/local/cuda/bin/:/opt/miniconda3/bin 44 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64 45 | 46 | %post 47 | # This file contains all custom installations, which was installed during the post phase of building the container. It avoids errors caused by double installation, if the container is build again with a changed configuration. 48 | if [ ! -f /opt/installed_tools.txt ]; then 49 | touch /opt/installed_tools.txt 50 | fi 51 | 52 | keep_src_and_build=1 53 | 54 | ############################################################################################ 55 | ### install software from package manager 56 | ############################################################################################ 57 | apt update 58 | apt install -y nano python3 wget git pkg-config uuid-dev gdb locales locales-all 59 | 60 | ############################################################################################ 61 | ### configure locales (otherwise error at test Prompt/ValuePrinter/Strings.C) 62 | ############################################################################################ 63 | locale-gen en_US.UTF-8 64 | update-locale LANG=en_US.UTF-8 65 | 66 | ############################################################################################ 67 | ### read config and set environment variables 68 | ############################################################################################ 69 | XCC_PROJECT_PATH=$(cat /opt/config.json | python3 -c "import sys, json; print(json.load(sys.stdin)['XCC_PROJECT_PATH'])") 70 | XCC_BUILD_TYPE=$(cat /opt/config.json | python3 -c "import sys, json; print(json.load(sys.stdin)['XCC_BUILD_TYPE'])") 71 | XCC_NUM_THREADS=$(cat /opt/config.json | python3 -c "import sys, json; print(json.load(sys.stdin)['XCC_NUM_THREADS'])") 72 | 73 | # remove environment variables from the last build process 74 | echo "" > $SINGULARITY_ENVIRONMENT 75 | echo "export XCC_PROJECT_PATH=$XCC_PROJECT_PATH" >>$SINGULARITY_ENVIRONMENT 76 | echo "export XCC_BUILD_TYPE=$XCC_BUILD_TYPE" >>$SINGULARITY_ENVIRONMENT 77 | echo "export XCC_NUM_THREADS=$XCC_NUM_THREADS" >>$SINGULARITY_ENVIRONMENT 78 | 79 | ############################################################################################ 80 | ### prepare installation from source 81 | ############################################################################################ 82 | mkdir -p /opt/tmp 83 | cd /opt/tmp 84 | 85 | ############################################################################################ 86 | ### install cmake v 3.14 87 | ############################################################################################ 88 | if ! grep -q "cmake_3.14" "/opt/installed_tools.txt"; then 89 | cd /opt 90 | apt remove -y cmake 91 | wget https://github.com/Kitware/CMake/releases/download/v3.14.3/cmake-3.14.3-Linux-x86_64.tar.gz 92 | tar -xzf cmake-3.14.3-Linux-x86_64.tar.gz 93 | ln -s /opt/cmake-3.14.3-Linux-x86_64/bin/cmake /usr/bin/cmake 94 | ln -s /opt/cmake-3.14.3-Linux-x86_64/bin/ccmake /usr/bin/ccmake 95 | rm cmake-3.14.3-Linux-x86_64.tar.gz 96 | cd /opt/tmp 97 | echo "cmake_3.14" >> /opt/installed_tools.txt 98 | fi 99 | 100 | # jupyter notebook/lab needs the folder /run/user and singularity doesn't generate one 101 | if [ ! -d /run/user ]; then 102 | mkdir /run/user 103 | chmod 777 /run/user 104 | fi 105 | 106 | ############################################################################################ 107 | ### install xeus 108 | ############################################################################################ 109 | if ! grep -q "libzmq" "/opt/installed_tools.txt"; then 110 | git clone --depth 1 --branch v4.2.5 https://github.com/zeromq/libzmq.git 111 | mkdir libzmq/build 112 | cd libzmq/build 113 | cmake -D WITH_PERF_TOOL=OFF -D ZMQ_BUILD_TESTS=OFF -D ENABLE_CPACK=OFF -D CMAKE_BUILD_TYPE=$XCC_BUILD_TYPE .. 114 | make -j$XCC_NUM_THREADS 115 | make install -j$XCC_NUM_THREADS 116 | cd ../.. 117 | if [ $keep_src_and_build == 0 ]; then 118 | rm -r libzmq 119 | fi 120 | echo "libzmq" >> /opt/installed_tools.txt 121 | fi 122 | 123 | if ! grep -q "cppzmq" "/opt/installed_tools.txt"; then 124 | git clone --depth 1 --branch v4.3.0 https://github.com/zeromq/cppzmq.git 125 | mkdir cppzmq/build 126 | cd cppzmq/build 127 | cmake -D CMAKE_BUILD_TYPE=$XCC_BUILD_TYPE .. 128 | make install -j$XCC_NUM_THREADS 129 | cd ../.. 130 | if [ $keep_src_and_build == 0 ]; then 131 | rm -r cppzmq 132 | fi 133 | echo "cppzmq" >> /opt/installed_tools.txt 134 | fi 135 | 136 | if ! grep -q "nlohmann_json" "/opt/installed_tools.txt"; then 137 | git clone --depth 1 --branch v3.3.0 https://github.com/nlohmann/json.git 138 | mkdir json/build 139 | cd json/build 140 | cmake -D CMAKE_BUILD_TYPE=$XCC_BUILD_TYPE .. 141 | make install -j$XCC_NUM_THREADS 142 | cd ../.. 143 | if [ $keep_src_and_build == 0 ]; then 144 | rm -r json 145 | fi 146 | echo "nlohmann_json" >> /opt/installed_tools.txt 147 | fi 148 | 149 | if ! grep -q "xtl" "/opt/installed_tools.txt"; then 150 | git clone --depth 1 --branch 0.6.5 https://github.com/QuantStack/xtl.git 151 | mkdir xtl/build 152 | cd xtl/build 153 | cmake -D CMAKE_BUILD_TYPE=$XCC_BUILD_TYPE .. 154 | make install -j$XCC_NUM_THREADS 155 | cd ../.. 156 | if [ $keep_src_and_build == 0 ]; then 157 | rm -r xtl 158 | fi 159 | echo "xtl" >> /opt/installed_tools.txt 160 | fi 161 | 162 | if ! grep -q "openssl" "/opt/installed_tools.txt"; then 163 | wget https://www.openssl.org/source/openssl-1.1.1c.tar.gz 164 | tar -xvzf openssl-1.1.1c.tar.gz 165 | cd openssl-1.1.1c 166 | ./config 167 | make -j$XCC_NUM_THREADS 168 | make -j$XCC_NUM_THREADS install 169 | cd .. 170 | rm -r openssl-1.1.1c openssl-1.1.1c.tar.gz 171 | echo "openssl" >> /opt/installed_tools.txt 172 | fi 173 | 174 | if ! grep -q "xeus" "/opt/installed_tools.txt"; then 175 | git clone --depth 1 --branch 0.20.0 https://github.com/QuantStack/xeus.git 176 | mkdir xeus/build 177 | cd xeus/build 178 | cmake -D BUILD_EXAMPLES=OFF -D DISABLE_ARCH_NATIVE=ON -D CMAKE_BUILD_TYPE=$XCC_BUILD_TYPE .. 179 | make -j$XCC_NUM_THREADS 180 | make install -j$XCC_NUM_THREADS 181 | cd ../.. 182 | if [ $keep_src_and_build == 0 ]; then 183 | rm -r xeus 184 | fi 185 | echo "xeus" >> /opt/installed_tools.txt 186 | fi 187 | 188 | ############################################################################################ 189 | ### install xeus-cling dependencies 190 | ############################################################################################ 191 | if ! grep -q "pugixml" "/opt/installed_tools.txt"; then 192 | git clone --depth 1 --branch v1.8.1 https://github.com/zeux/pugixml.git 193 | mkdir pugixml/build 194 | cd pugixml/build 195 | cmake -D CMAKE_BUILD_TYPE=$XCC_BUILD_TYPE -D CMAKE_POSITION_INDEPENDENT_CODE=ON .. 196 | make install -j$XCC_NUM_THREADS 197 | cd ../.. 198 | if [ $keep_src_and_build == 0 ]; then 199 | rm -r pugixml 200 | fi 201 | echo "pugixml" >> /opt/installed_tools.txt 202 | fi 203 | 204 | if ! grep -q "cxxopts" "/opt/installed_tools.txt"; then 205 | git clone --depth 1 --branch v2.1.1 https://github.com/jarro2783/cxxopts.git 206 | mkdir cxxopts/build 207 | cd cxxopts/build 208 | cmake -D CMAKE_BUILD_TYPE=$XCC_BUILD_TYPE .. 209 | make install -j$XCC_NUM_THREADS 210 | cd ../.. 211 | if [ $keep_src_and_build == 0 ]; then 212 | rm -r cxxopts 213 | fi 214 | echo "cxxopts" >> /opt/installed_tools.txt 215 | fi 216 | 217 | # the link is necessary, to add the miniconda3/bin folder to PATH 218 | ln -fs $XCC_PROJECT_PATH/miniconda3/ /opt/miniconda3 219 | 220 | if [ $keep_src_and_build == 0 ]; then 221 | rm -r /opt/tmp 222 | fi 223 | 224 | %runscript 225 | # run all user-space install scripts 226 | sh $SCIF_APPRUN_install_miniconda3 227 | sh $SCIF_APPRUN_build_cling 228 | sh $SCIF_APPRUN_build_xeus_cling 229 | 230 | %apphelp install_miniconda3 231 | This script install miniconda3 in your project directory. 232 | 233 | %apprun install_miniconda3 234 | ############################################################################################ 235 | ### install miniconda 3 236 | ############################################################################################ 237 | if [ ! -d $XCC_PROJECT_PATH ]; then 238 | mkdir -p $XCC_PROJECT_PATH 239 | fi 240 | 241 | if [ -d $XCC_PROJECT_PATH/miniconda3 ]; then 242 | echo "ERROR: $XCC_PROJECT_PATH/miniconda3 already exists.\nCheck if Miniconda is already installed." 243 | exit 255 244 | else 245 | cd $XCC_PROJECT_PATH 246 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh 247 | chmod u+x Miniconda3-latest-Linux-x86_64.sh 248 | ./Miniconda3-latest-Linux-x86_64.sh -b -p $XCC_PROJECT_PATH/miniconda3 249 | $XCC_PROJECT_PATH/miniconda3/bin/conda install -y jupyter 250 | $XCC_PROJECT_PATH/miniconda3/bin/conda install -y -c conda-forge jupyterlab 251 | rm Miniconda3-latest-Linux-x86_64.sh 252 | fi 253 | 254 | %apphelp 255 | Download and build clang from source. If cling already exists, just do "make". 256 | 257 | %apprun build_cling 258 | ############################################################################################ 259 | ### build cling 260 | ############################################################################################ 261 | if [ ! -d $XCC_PROJECT_PATH ]; then 262 | mkdir -p $XCC_PROJECT_PATH 263 | fi 264 | 265 | if [ ! -d $XCC_PROJECT_PATH/cling ]; then 266 | mkdir -p $XCC_PROJECT_PATH/cling 267 | mkdir -p $XCC_PROJECT_PATH/cling/build 268 | mkdir -p $XCC_PROJECT_PATH/cling/install 269 | cd $XCC_PROJECT_PATH/cling 270 | git clone --branch cling-patches http://root.cern.ch/git/llvm.git src 271 | cd src/tools 272 | git clone --branch test_release https://github.com/SimeonEhrig/cling 273 | git clone --branch cling-patches http://root.cern.ch/git/clang.git 274 | cd $XCC_PROJECT_PATH/cling/build 275 | cmake ../src/ -DCMAKE_BUILD_TYPE=$XCC_BUILD_TYPE -DBUILD_SHARED_LIBS=OFF -DLLVM_ABI_BREAKING_CHECKS="FORCE_OFF" -DCMAKE_LINKER=/usr/bin/gold -DCMAKE_INSTALL_PREFIX=$XCC_PROJECT_PATH/cling/install -DLLVM_ENABLE_RTTI=ON 276 | make -j$XCC_NUM_THREADS 277 | make install -j$XCC_NUM_THREADS 278 | else 279 | cd $XCC_PROJECT_PATH/cling/build 280 | make -j$XCC_NUM_THREADS 281 | make install -j$XCC_NUM_THREADS 282 | fi 283 | 284 | %apphelp 285 | Install xeus-cling from source. If Miniconda is not installed, the installation cancels. If xeus-cling already exists, just do "make". 286 | 287 | %apprun build_xeus-cling 288 | ############################################################################################ 289 | ### build xeus-cling 290 | ############################################################################################ 291 | if [ ! -d $XCC_PROJECT_PATH ]; then 292 | mkdir -p $XCC_PROJECT_PATH 293 | fi 294 | 295 | if [ ! -d $XCC_PROJECT_PATH/miniconda3 ]; then 296 | echo "ERROR: Miniconda not found. Please install miniconda." 297 | exit 255 298 | fi 299 | 300 | if [ ! -d $XCC_PROJECT_PATH/cling ]; then 301 | echo "ERROR: Cling not found. Please install Cling." 302 | exit 255 303 | fi 304 | 305 | PATH=$PATH:$XCC_PROJECT_PATH/cling/install/bin 306 | LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$XCC_PROJECT_PATH/cling/install/lib 307 | 308 | if [ ! -d $XCC_PROJECT_PATH/xeus-cling ]; then 309 | # check if folder with xeus-cling cuda kernel definitions exists 310 | XCC_KERNEL_PATH=$PWD/jupyter_kernels 311 | if [ ! -d $XCC_KERNEL_PATH ]; then 312 | echo "Error: Can not find folder jupyter_kernels, which contains the kernel definitions for xeus-cling-cuda.\nPlease run singularity in the same folder as the jupyter_kernels folder or set the path to the jupyter_kernels folder with: singularity run --app build_xeus-cling --pwd /path/to/jupyter_kernels image.sing" 313 | exit 255 314 | fi 315 | 316 | cd $XCC_PROJECT_PATH 317 | git clone --branch 0.6.0 https://github.com/QuantStack/xeus-cling.git 318 | cd xeus-cling 319 | # small workaround, otherwise the build type is always set to release 320 | sed -i -e "s|set(CMAKE_BUILD_TYPE Release CACHE STRING \"Choose the type of build.\" FORCE)|#set(CMAKE_BUILD_TYPE Release CACHE STRING \"Choose the type of build.\" FORCE)|" CMakeLists.txt 321 | cd .. 322 | mkdir xeus-cling/build 323 | cd xeus-cling/build 324 | cmake -DCMAKE_INSTALL_PREFIX=$XCC_PROJECT_PATH/miniconda3/ -DCMAKE_INSTALL_LIBDIR=$XCC_PROJECT_PATH/miniconda3/lib -DCMAKE_BUILD_TYPE=$XCC_BUILD_TYPE -DCMAKE_CXX_FLAGS="-I $XCC_PROJECT_PATH/cling/install/include" -DCMAKE_LINKER=/usr/bin/gold -DDISABLE_ARCH_NATIVE=ON .. 325 | make -j$XCC_NUM_THREADS 326 | make install -j$XCC_NUM_THREADS 327 | 328 | # specialize the xeus-cling cuda configurations depending of your build configuration and install they afterwards 329 | for std in 11 14 17 330 | do 331 | sed -e "s||${XCC_PROJECT_PATH}/miniconda3/bin/xcpp|" -e "s||-include${XCC_PROJECT_PATH}/miniconda3/include/xcpp/xmime.hpp|" $XCC_KERNEL_PATH/xeus-cling-cpp${std}-cuda/kernel.json.in > $XCC_KERNEL_PATH/xeus-cling-cpp${std}-cuda/kernel.json 332 | jupyter-kernelspec install --user $XCC_KERNEL_PATH/xeus-cling-cpp${std}-cuda 333 | done 334 | else 335 | cd $XCC_PROJECT_PATH/xeus-cling/build 336 | make -j$XCC_NUM_THREADS 337 | make install -j$XCC_NUM_THREADS 338 | fi 339 | -------------------------------------------------------------------------------- /deprecated/jupyter_kernels/kernels.tar: -------------------------------------------------------------------------------- 1 | xeus-cling-cpp11-cuda/0000755000175200017530000000000013441722753014331 5ustar sehrigsehrigxeus-cling-cpp11-cuda/kernel.json.in0000644000175200017530000000025613376766375017133 0ustar sehrigsehrig{ 2 | "display_name": "C++11-CUDA", 3 | "argv": [ 4 | "", 5 | "-f", 6 | "{connection_file}", 7 | "-std=c++11", 8 | "-xcuda" 9 | ], 10 | "language": "C++11" 11 | } 12 | xeus-cling-cpp14-cuda/0000775000175200017530000000000013441722764014340 5ustar sehrigsehrigxeus-cling-cpp14-cuda/kernel.json.in0000644000175200017530000000025613376766361017131 0ustar sehrigsehrig{ 13 | "display_name": "C++14-CUDA", 14 | "argv": [ 15 | "", 16 | "-f", 17 | "{connection_file}", 18 | "-std=c++14", 19 | "-xcuda" 20 | ], 21 | "language": "C++14" 22 | } 23 | xeus-cling-cpp17-cuda/0000775000175200017530000000000013441722777014347 5ustar sehrigsehrigxeus-cling-cpp17-cuda/kernel.json.in0000644000175200017530000000025613376766413017132 0ustar sehrigsehrig{ 24 | "display_name": "C++17-CUDA", 25 | "argv": [ 26 | "", 27 | "-f", 28 | "{connection_file}", 29 | "-std=c++17", 30 | "-xcuda" 31 | ], 32 | "language": "C++17" 33 | } 34 | -------------------------------------------------------------------------------- /deprecated/jupyter_kernels/xeus-cling-cpp11-cuda/kernel.json.in: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "C++11-CUDA", 3 | "argv": [ 4 | "", 5 | "-f", 6 | "{connection_file}", 7 | "-std=c++11", 8 | "-xcuda", 9 | "" 10 | ], 11 | "language": "C++11" 12 | } 13 | -------------------------------------------------------------------------------- /deprecated/jupyter_kernels/xeus-cling-cpp14-cuda/kernel.json.in: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "C++14-CUDA", 3 | "argv": [ 4 | "", 5 | "-f", 6 | "{connection_file}", 7 | "-std=c++14", 8 | "-xcuda", 9 | "" 10 | ], 11 | "language": "C++14" 12 | } 13 | -------------------------------------------------------------------------------- /deprecated/jupyter_kernels/xeus-cling-cpp17-cuda/kernel.json.in: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "C++17-CUDA", 3 | "argv": [ 4 | "", 5 | "-f", 6 | "{connection_file}", 7 | "-std=c++17", 8 | "-xcuda", 9 | "" 10 | ], 11 | "language": "C++17" 12 | } 13 | -------------------------------------------------------------------------------- /deprecated/kdevelop/README.md: -------------------------------------------------------------------------------- 1 | # Kdevelop extension for dev-xeus-cling-cuda 2 | 3 | This container extends the dev-xeus-cling-cuda.sing container with a kdevelop IDE. The extension is completely optional. 4 | 5 | # About 6 | Cling is a really huge project. For effective development a is IDE really useful. But a lot of IDEs has problems, to handle big project like cling, clang/llvm or the linux kernel. Kdevelop is really efficient and can handle this project size but it also needs some configuration. 7 | 8 | Beside, often I develop on powerful workstation systems via ssh. So, using a IDE is not easy, because you have to handle the distribution of graphical output and source code and build environment. 9 | 10 | As result of this two points, I develop this extension, to use my complete development stack including the IDE on a remote system. It's also a documentation of the configuration and easy possibility to change my development system. 11 | 12 | # Installation 13 | 14 | At first, you have to [build](../README.md) the `dev-xeus-cling-cuda.sing` container. Afterwards, run the following commands. 15 | ```bash 16 | #cd xeus-cling-cuda-container/kdevelop 17 | ./config_generator.sh 18 | sudo singularity build kdev-xeus-cling-cuda.sing kdev-xeus-cling-cuda 19 | ``` 20 | Now, the container with kdevelop is ready. Important, the container just works with the current user. If different users want to use kdevelop, they all have to build his own container. 21 | 22 | ## Building the Container on another System 23 | 24 | If you have no root permission on your development system, you can build the container on a another system. In this case, you can generate the `kdev_config.json` on you development system an copy it to the system, which you want to build the container. Alternative, you can write the `kdev_config.json` by yourself. The content is the follow: 25 | 26 | ``` json 27 | { 28 | "XCC_USER_ID" : "" 29 | } 30 | ``` 31 | To get the user, simply run `id -u` on your development system. 32 | 33 | # Usage 34 | 35 | For easy usage, there are two scripts in the folder: `run_shell.sh` and `run_kdevelop.sh`. The first script starts a interactive shell season of the container. The second script starts kdevelop direct. 36 | If you want to use custom commands, you have to use the parameter `-B /run/user/$(id -u):/run/user/$(id -u)` in your `singularity` command. This binding command allows to 'redirect' the graphical output of the container to the host. 37 | 38 | # Configuration of Kdevelop for the Cling Project 39 | 40 | Before you load in the cling project in kdevelop go to `Settings->Configure KDevelop->LanguageSupport->Background Parser` and disable the Background Parser. Otherwise the parser will crash at parsing a test case file from clang. After this, import the cling source code via `Project->Open / Import Project`. Then click right on the project root in the project browser and choose `Open Configuration...`. Afterwards, choose `Project Filter` in the configuration window and add an new rule. Add to the `pattern` the path `tools/clang/test`. The other properties should be `Files and Folders` and `Exclude`. Now you can re-enable the Background Parser. Now, you will see a progress bar on the lower right corner. 41 | 42 | Parsing the whole code takes a lot of time. On my Intel 14 Core System, I need about 20 minutes. Sometimes, the parser just parse with one core. In this case, simple restart kdevelop. Unfortunately, the parsing process isn't interruptable, so you have to do the process in one session. The parsing process have to run one times. Afterwards, the index will be loaded from disk and auto completion is avail after few seconds. 43 | -------------------------------------------------------------------------------- /deprecated/kdevelop/config_generator.sh: -------------------------------------------------------------------------------- 1 | echo "your user id is: $(id -u)" 2 | 3 | XCC_USER_ID=$(id -u) 4 | 5 | echo "{" > kdev_config.json 6 | echo -e "\t\"XCC_USER_ID\" : \"$XCC_USER_ID\"" >> kdev_config.json 7 | echo "}" >> kdev_config.json 8 | -------------------------------------------------------------------------------- /deprecated/kdevelop/kdev-xeus-cling-cuda: -------------------------------------------------------------------------------- 1 | Bootstrap: localimage 2 | From: ../dev-xeus-cling-cuda.sing 3 | 4 | %help 5 | The container extends the dev-xeus-cling-cuda container with the IDE kdevelop. 6 | 7 | %setup 8 | 9 | %files 10 | kdev_config.json /opt/kdev_config.json 11 | 12 | %labels 13 | Maintainer Simeon Ehrig 14 | Email simeonehrig@web.de 15 | Version 1.0 16 | 17 | %environment 18 | export PATH=$PATH:/opt/KDevelop_5.3 19 | 20 | %post 21 | # This file contains all custom installations, which was installed during the post phase of building the container. It avoids errors caused by double installation, if the container is build again with a changed configuration. 22 | if [ ! -f /opt/installed_tools.txt ]; then 23 | touch /opt/installed_tools.txt 24 | fi 25 | 26 | ############################################################################################ 27 | ### install software from package manager 28 | ############################################################################################ 29 | apt update 30 | apt install -y qt5-default xterm 31 | 32 | ############################################################################################ 33 | ### read config and set environment variables 34 | ############################################################################################ 35 | XCC_USER_ID=$(cat /opt/kdev_config.json | python3 -c "import sys, json; print(json.load(sys.stdin)['XCC_USER_ID'])") 36 | 37 | if ! grep -q "export XCC_USER_ID=$XCC_USER_ID" "$SINGULARITY_ENVIRONMENT"; then 38 | echo "export XCC_USER_ID=$XCC_USER_ID" >>$SINGULARITY_ENVIRONMENT 39 | fi 40 | 41 | # kdevelop needs this folder to bind the graphical output with the host 42 | if [ ! -d /run/user/$XCC_USER_ID ]; then 43 | mkdir /run/user/$XCC_USER_ID 44 | fi 45 | 46 | ############################################################################################ 47 | ### install gdb 8.2 48 | ############################################################################################ 49 | if ! grep -q "gdb82" "/opt/installed_tools.txt"; then 50 | apt purge -y gdb 51 | apt install -y texinfo checkinstall 52 | cd /opt 53 | wget http://ftp.gnu.org/gnu/gdb/gdb-8.2.tar.xz 54 | tar -xf gdb-8.2.tar.xz 55 | cd gdb-8.2 56 | ./configure 57 | make -j 58 | #cd gdb 59 | make -j install 60 | cd /opt 61 | rm -r gdb-8.2 gdb-8.2.tar.xz 62 | echo "gdb82" >> /opt/installed_tools.txt 63 | fi 64 | 65 | ############################################################################################ 66 | ### install KDevelop 5.3 67 | ############################################################################################ 68 | if ! grep -q "kdevelop53" "/opt/installed_tools.txt"; then 69 | cd /opt 70 | wget -O KDevelop.AppImage https://download.kde.org/stable/kdevelop/5.3.0/bin/linux/KDevelop-5.3.0-x86_64.AppImage 71 | chmod u+x KDevelop.AppImage 72 | ./KDevelop.AppImage --appimage-extract 73 | mv squashfs-root/ KDevelop_5.3/ 74 | ln -s /opt/KDevelop_5.3/AppRun /opt/KDevelop_5.3/kdevelop 75 | chmod -R 707 /opt/KDevelop_5.3 76 | rm /opt/KDevelop.AppImage 77 | echo "kdevelop53" >> /opt/installed_tools.txt 78 | fi 79 | 80 | %runscript -------------------------------------------------------------------------------- /deprecated/kdevelop/run_kdevelop.sh: -------------------------------------------------------------------------------- 1 | singularity exec --nv -B /run/user/$(id -u):/run/user/$(id -u) kdev-xeus-cling-cuda.sing kdevelop 2 | -------------------------------------------------------------------------------- /deprecated/kdevelop/run_shell.sh: -------------------------------------------------------------------------------- 1 | singularity shell --nv -B /run/user/$(id -u):/run/user/$(id -u) kdev-xeus-cling-cuda.sing 2 | -------------------------------------------------------------------------------- /dev_container.py: -------------------------------------------------------------------------------- 1 | """Script to a Singularity development container of the xeus-cling-cuda stack. 2 | 3 | run `python dev_container.py --help` to get container generation options 4 | 5 | the script requires hpccm (https://github.com/NVIDIA/hpc-container-maker) 6 | 7 | the script is designed to be executed standalone 8 | """ 9 | 10 | import argparse 11 | import sys 12 | import os 13 | import xcc.generator as gn 14 | 15 | 16 | def main(): 17 | ################################################################## 18 | # parse args 19 | ################################################################## 20 | parser = argparse.ArgumentParser( 21 | description='Script to generate Singularity receipt for a xeus-cling-cuda development container', 22 | formatter_class=argparse.RawTextHelpFormatter) 23 | parser.add_argument( 24 | '-j', type=str, help='number of build threads for make (default: -j)') 25 | parser.add_argument( 26 | '-l', type=str, help='number of linker threads for the cling build (default: -j)') 27 | parser.add_argument('-o', '--out', type=str, 28 | help='set path of output file (default: stdout)\n' + 29 | 'if --store_gen_command is set to 1, save file with arguments beside the recipe') 30 | parser.add_argument('-b', type=str, default='', 31 | choices=['DEBUG', 'RELEASE', 32 | 'RELWITHDEBINFO', 'MINSIZEREL'], 33 | help='set the CMAKE_BUILD_TYPE (default: DEBUG)') 34 | parser.add_argument('--second_build', type=str, default='', 35 | choices=['', 'DEBUG', 'RELEASE', 36 | 'RELWITHDEBINFO', 'MINSIZEREL'], 37 | help='If set, create a second cling build. Can be used, for example, if you want to get a debug and release version of cling.') 38 | parser.add_argument('--build_command', action='store_true', 39 | help='print the build command for the container') 40 | parser.add_argument('--run_command', action='store_true', 41 | help='print the run command for the container') 42 | parser.add_argument('--build_prefix', type=str, default='/tmp', 43 | help='Set source and build path of the libraries and projects.\n' 44 | 'Only the prefixes /tmp (build outside the container with singularity) and /opt are allowed (default: /tmp).' 45 | 'Run --help_build_dir to get more information.') 46 | parser.add_argument('--project_path', type=str, default=os.getcwd() + '/build', 47 | help='Set prefix folder of Miniconda, Cling and Xeus-Cling. Default: $(pwd)/build') 48 | parser.add_argument('--keep_build', action='store_true', 49 | help='keep source and build files after installation\n') 50 | parser.add_argument('--help_build_prefix', action='store_true', 51 | help='get information about build process') 52 | parser.add_argument('--clang_version', type=int, default=8, 53 | choices=[8, 9], 54 | help='set the version of the clang project compiler (default: 8)') 55 | parser.add_argument('--store_gen_command', type=int, default=1, 56 | choices=[0, 1], 57 | help='save the command with which the recipe was generated (default: 1)') 58 | parser.add_argument('--cling_url', type=str, 59 | help='Set custom Cling GitHub url.') 60 | parser.add_argument('--cling_branch', type=str, 61 | help='Used only when --cling_url is set. Change the GitHub branch of Cling. Cling GitHub Commit Hash is deleted.') 62 | parser.add_argument('--cling_hash', type=str, 63 | help='Used only when --cling_url is set. Change the GitHub Commit of Cling. Cling GitHub branch is deleted.') 64 | parser.add_argument('--build_libcxx', action='store_true', 65 | help='Set the flag to build the whole stack with libc++. ' 66 | 'Also add the libc++ and libc++abi projects to the llvm build.') 67 | 68 | args = parser.parse_args() 69 | 70 | ################################################################## 71 | # print help for building and running the container 72 | ################################################################## 73 | if args.build_command: 74 | print('build the container') 75 | print('singularity build --fakeroot .sif .def') 76 | print('download and build the source code to be further developed') 77 | print('singularity run .sif') 78 | sys.exit() 79 | 80 | if args.run_command: 81 | print('singularity exec --nv -B /run/user/$(id -u):/run/user/$(id -u) .sif jupyter-lab') 82 | sys.exit() 83 | 84 | ################################################################## 85 | # print help for build 86 | ################################################################## 87 | 88 | if args.help_build_prefix: 89 | print('Singularity: The folders /tmp and $HOME from the host are automatically mounted at build time') 90 | print(' This can be used to cache builds. But it also can cause problems. To avoid problems,') 91 | print(' you should delete the source and build folders after building the container.') 92 | print(' If you you want to keep the build inside the container, you should choose an unbound') 93 | print(' path. For example /opt') 94 | sys.exit() 95 | 96 | # parse number of build threads 97 | # if no threads are set, it is set to None which means it is executed with -j 98 | if args.j: 99 | threads = int(args.j) 100 | if threads < 1: 101 | raise ValueError('-j have to be greater than 0') 102 | else: 103 | threads = None 104 | 105 | if args.l: 106 | linker_threads = int(args.l) 107 | if linker_threads < 1: 108 | raise ValueError('-l have to be greater than 0') 109 | else: 110 | linker_threads = None 111 | 112 | # depending on the path, certain build locations can be difficult 113 | build_prefix = str(args.build_prefix) 114 | if (not build_prefix.startswith('/tmp')) and (not build_prefix.startswith('/opt')): 115 | raise ValueError('--build_prefix have to start with /tmp or /opt') 116 | 117 | gen_args=None 118 | if args.store_gen_command == 1: 119 | gen_args=' '.join(sys.argv) 120 | if args.out: 121 | with open(os.path.dirname(os.path.abspath(args.out)) + '/' + 122 | os.path.splitext(os.path.basename(args.out))[0] + '_command.txt', 'w') as filehandle: 123 | filehandle.write(gen_args) 124 | 125 | xcc_gen = gn.XCC_gen(container='singularity', 126 | build_prefix=build_prefix, 127 | install_prefix='/usr/local', 128 | build_type=args.b, 129 | keep_build=args.keep_build, 130 | threads=threads, 131 | linker_threads=linker_threads, 132 | clang_version=args.clang_version, 133 | gen_args=gen_args, 134 | build_libcxx=args.build_libcxx) 135 | 136 | if args.cling_url: 137 | if args.cling_branch is not None and args.cling_hash is not None: 138 | print('--cling_branch and --cling_hash cannot be used at the same time') 139 | exit(1) 140 | else: 141 | xcc_gen.cling_url = args.cling_url 142 | xcc_gen.cling_branch = args.cling_branch 143 | xcc_gen.cling_hash = args.cling_hash 144 | 145 | stage = xcc_gen.gen_devel_stage(project_path=os.path.abspath(args.project_path), 146 | dual_build_type = (None if args.second_build == '' else args.second_build)) 147 | 148 | ################################################################## 149 | # write to file or stdout 150 | ################################################################## 151 | if args.out: 152 | with open(args.out, 'w') as filehandle: 153 | filehandle.write(stage.__str__()) 154 | else: 155 | print(stage) 156 | 157 | 158 | if __name__ == "__main__": 159 | main() 160 | -------------------------------------------------------------------------------- /rel_container.py: -------------------------------------------------------------------------------- 1 | """Script to generate Docker or Singularity 2 | xeus-cling-cuda release container 3 | 4 | run `python rel_container.py --help` to get container generation options 5 | 6 | the script requires hpccm (https://github.com/NVIDIA/hpc-container-maker) 7 | 8 | the script is designed to be executed standalone 9 | """ 10 | 11 | import argparse 12 | import json 13 | import sys 14 | import os 15 | import xcc.generator as gn 16 | 17 | 18 | def main(): 19 | ################################################################## 20 | # parse args 21 | ################################################################## 22 | parser = argparse.ArgumentParser( 23 | description='Script to generate a Dockerfile or Singularity receipt for xeus-cling-cuda', 24 | formatter_class=argparse.RawTextHelpFormatter) 25 | parser.add_argument('--container', type=str, default='singularity', 26 | choices=['docker', 'singularity'], 27 | help='generate receipt for docker or singularity (default: singularity)') 28 | parser.add_argument( 29 | '-j', type=str, help='number of build threads for make (default: -j)') 30 | parser.add_argument( 31 | '-l', type=str, help='number of linker threads for the cling build (default: -j)') 32 | parser.add_argument('-o', '--out', type=str, 33 | help='set path of output file (default: stdout)' + 34 | 'if --store_gen_command is set to 1, save file with arguments beside the recipe') 35 | parser.add_argument('--build_command', type=str, 36 | choices=['docker', 'singularity'], 37 | help='print the build command for the container') 38 | parser.add_argument('--run_command', type=str, 39 | choices=['docker', 'singularity'], 40 | help='print the run command for the container') 41 | parser.add_argument('--build_prefix', type=str, default='/tmp', 42 | help='Set source and build path of the libraries and projects.\n' 43 | 'Only the prefixes /tmp (build outside the container with singularity) and /opt are allowed (default: /tmp).' 44 | 'Run --help_build_dir to get more information.') 45 | parser.add_argument('--keep_build', action='store_true', 46 | help='keep source and build files after installation\n' 47 | 'only for singularity supported, because it can store builds in the host memory\n' 48 | 'for docker, it is not useful, because the multi-stage build') 49 | parser.add_argument('--help_build_prefix', action='store_true', 50 | help='get information about build process') 51 | parser.add_argument('--clang_version', type=int, default=8, 52 | choices=[8, 9], 53 | help='set the version of the clang project compiler (default: 8)') 54 | parser.add_argument('--store_gen_command', type=int, default=1, 55 | choices=[0, 1], 56 | help='save the command with which the recipe was generated (default: 1)') 57 | parser.add_argument('--cling_url', type=str, 58 | help='Set custom Cling GitHub url.') 59 | parser.add_argument('--cling_branch', type=str, 60 | help='Used only when --cling_url is set. Change the GitHub branch of Cling. Cling GitHub Commit Hash is deleted.') 61 | parser.add_argument('--cling_hash', type=str, 62 | help='Used only when --cling_url is set. Change the GitHub Commit of Cling. Cling GitHub branch is deleted.') 63 | parser.add_argument('--build_libcxx', action='store_true', 64 | help='Set the flag to build the whole stack with libc++. ' 65 | 'Also add the libc++ and libc++abi projects to the llvm build.') 66 | 67 | args = parser.parse_args() 68 | 69 | ################################################################## 70 | # print help for building and running docker and singularity 71 | # container 72 | ################################################################## 73 | if args.build_command: 74 | if args.build_command == 'singularity': 75 | print('singularity build --fakeroot .sif .def') 76 | else: 77 | print('docker build -t hpccm_cling_cuda:dev .') 78 | sys.exit() 79 | 80 | if args.run_command: 81 | if args.run_command == 'singularity': 82 | print( 83 | 'singularity exec --nv -B /run/user/$(id -u):/run/user/$(id -u) .sif jupyter-lab') 84 | else: 85 | print( 86 | 'docker run --runtime=nvidia -p 8888:8888 --network="host" --rm -it hpccm_cling_cuda:dev') 87 | sys.exit() 88 | 89 | ################################################################## 90 | # print help for build 91 | ################################################################## 92 | 93 | if args.help_build_prefix: 94 | print('Docker: There are no automatic mounts at build time.') 95 | print(' To "cache" builds, you have to bind folders manual.') 96 | print(' See Singularity') 97 | print('Singularity: The folders /tmp and $HOME from the host are automatically mounted at build time') 98 | print(' This can be used to cache builds. But it also can cause problems. To avoid problems,') 99 | print(' you should delete the source and build folders after building the container.') 100 | print(' If you you want to keep the build inside the container, you should choose an unbound') 101 | print(' path. For example /opt') 102 | sys.exit() 103 | 104 | # parse number of build threads 105 | # if no threads are set, it is set to None which means it is executed with -j 106 | if args.j: 107 | threads = int(args.j) 108 | if threads < 1: 109 | raise ValueError('-j have to be greater than 0') 110 | else: 111 | threads = None 112 | 113 | if args.l: 114 | linker_threads = int(args.l) 115 | if linker_threads < 1: 116 | raise ValueError('-l have to be greater than 0') 117 | else: 118 | linker_threads = None 119 | 120 | # depending on the container software, certain build locations can be difficult 121 | build_prefix = str(args.build_prefix) 122 | if (not build_prefix.startswith('/tmp')) and (not build_prefix.startswith('/opt')): 123 | raise ValueError('--build_dir have to start with /tmp or /opt') 124 | 125 | gen_args=None 126 | if args.store_gen_command == 1: 127 | gen_args=' '.join(sys.argv) 128 | if args.out: 129 | with open(os.path.dirname(os.path.abspath(args.out)) + '/' + 130 | os.path.splitext(os.path.basename(args.out))[0] + '_command.txt', 'w') as filehandle: 131 | filehandle.write(gen_args) 132 | 133 | xcc_gen = gn.XCC_gen(container=args.container, 134 | build_prefix=build_prefix, 135 | keep_build=args.keep_build, 136 | threads=threads, 137 | linker_threads=linker_threads, 138 | clang_version=args.clang_version, 139 | gen_args=gen_args, 140 | build_libcxx=args.build_libcxx) 141 | 142 | if args.cling_url: 143 | if args.cling_branch is not None and args.cling_hash is not None: 144 | print('--cling_branch and --cling_hash cannot be used at the same time') 145 | exit(1) 146 | else: 147 | xcc_gen.cling_url = args.cling_url 148 | xcc_gen.cling_branch = args.cling_branch 149 | xcc_gen.cling_hash = args.cling_hash 150 | 151 | stage = xcc_gen.gen_release_single_stage() 152 | 153 | ################################################################## 154 | # write to file or stdout 155 | ################################################################## 156 | if args.out: 157 | with open(args.out, 'w') as filehandle: 158 | filehandle.write(stage.__str__()) 159 | else: 160 | print(stage) 161 | 162 | 163 | if __name__ == "__main__": 164 | main() 165 | -------------------------------------------------------------------------------- /xcc/__init__.py: -------------------------------------------------------------------------------- 1 | #import xcc.generator 2 | -------------------------------------------------------------------------------- /xcc/basestage.py: -------------------------------------------------------------------------------- 1 | """Function to generate base stage for the container 2 | 3 | """ 4 | 5 | import hpccm 6 | from hpccm.primitives import baseimage, label, environment, shell 7 | from hpccm.building_blocks.packages import packages 8 | from hpccm.building_blocks.cmake import cmake 9 | from hpccm.building_blocks.llvm import llvm 10 | 11 | import xcc.config 12 | 13 | 14 | def gen_base_stage(config: xcc.config.XCC_Config) -> hpccm.Stage: 15 | """Returns an nvidia cuda container stage, which has some basic configuration. 16 | 17 | * labels are set 18 | * software via apt installed 19 | * set language to en_US.UTF-8 20 | * install modern cmake version 21 | * create folder /run/user 22 | 23 | :param config: Configuration object, which contains different information for the stage 24 | :type config: xcc.config.XCC_Config 25 | :returns: hpccm Stage 26 | :rtype: hpccm.Stage 27 | 28 | """ 29 | hpccm.config.set_container_format(config.container) 30 | if config.container == "singularity": 31 | hpccm.config.set_singularity_version("3.3") 32 | 33 | stage = hpccm.Stage() 34 | stage += baseimage(image="nvidia/cuda:8.0-devel-ubuntu16.04", _as="stage") 35 | 36 | stage += label( 37 | metadata={ 38 | "XCC Version": str(config.version), 39 | "Author": config.author, 40 | "E-Mail": config.email, 41 | } 42 | ) 43 | 44 | if config.gen_args: 45 | stage += environment(variables={"XCC_GEN_ARGS": '"' + config.gen_args + '"'}) 46 | 47 | # LD_LIBRARY_PATH is not taken over correctly when the docker container 48 | # is converted to a singularity container. 49 | stage += environment( 50 | variables={"LD_LIBRARY_PATH": "$LD_LIBRARY_PATH:/usr/local/cuda/lib64"} 51 | ) 52 | stage += environment(variables={"CMAKE_PREFIX_PATH": config.install_prefix}) 53 | stage += packages( 54 | ospackages=[ 55 | "git", 56 | "python", 57 | "wget", 58 | "pkg-config", 59 | "uuid-dev", 60 | "gdb", 61 | "locales", 62 | "locales-all", 63 | "unzip", 64 | ] 65 | ) 66 | # set language to en_US.UTF-8 to avoid some problems with the cling output system 67 | stage += shell( 68 | commands=["locale-gen en_US.UTF-8", "update-locale LANG=en_US.UTF-8"] 69 | ) 70 | 71 | stage += shell( 72 | commands=[ 73 | "", 74 | "#/////////////////////////////", 75 | "#// Install Clang and tools //", 76 | "#/////////////////////////////", 77 | ] 78 | ) 79 | 80 | # install clang/llvm 81 | # add ppa for modern clang/llvm versions 82 | stage += shell( 83 | commands=[ 84 | "wget http://llvm.org/apt/llvm-snapshot.gpg.key", 85 | "apt-key add llvm-snapshot.gpg.key", 86 | "rm llvm-snapshot.gpg.key", 87 | 'echo "" >> /etc/apt/sources.list', 88 | 'echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-' 89 | + str(config.clang_version) 90 | + ' main" >> /etc/apt/sources.list', 91 | 'echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-' 92 | + str(config.clang_version) 93 | + ' main" >> /etc/apt/sources.list', 94 | ] 95 | ) 96 | 97 | stage += llvm(version=str(config.clang_version)) 98 | # set clang 8 as compiler for all projects during container build time 99 | stage += shell( 100 | commands=[ 101 | "export CC=clang-" + str(config.clang_version), 102 | "export CXX=clang++-" + str(config.clang_version), 103 | ] 104 | ) 105 | 106 | # install clang development tools 107 | clang_extra = [ 108 | "clang-tidy-" + str(config.clang_version), 109 | "clang-tools-" + str(config.clang_version), 110 | ] 111 | 112 | # install libc++ and libc++abi depending of the clang version 113 | if config.build_libcxx: 114 | clang_extra += [ 115 | "libc++1-" + str(config.clang_version), 116 | "libc++-" + str(config.clang_version) + "-dev", 117 | "libc++abi1-" + str(config.clang_version), 118 | "libc++abi-" + str(config.clang_version) + "-dev", 119 | ] 120 | stage += packages(ospackages=clang_extra) 121 | 122 | stage += cmake(eula=True, version="3.18.0") 123 | 124 | # the folder is necessary for jupyter lab 125 | if config.container == "singularity": 126 | stage += shell(commands=["mkdir -p /run/user", "chmod 777 /run/user"]) 127 | 128 | # install ninja build system 129 | stage += shell( 130 | commands=[ 131 | "", 132 | "#/////////////////////////////", 133 | "#// Install Ninja //", 134 | "#/////////////////////////////", 135 | "cd /opt", 136 | "wget https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip", 137 | "unzip ninja-linux.zip", 138 | "mv ninja /usr/local/bin/", 139 | "rm ninja-linux.zip", 140 | "cd -", 141 | ] 142 | ) 143 | 144 | return stage 145 | -------------------------------------------------------------------------------- /xcc/cling.py: -------------------------------------------------------------------------------- 1 | """Function to create build instructions for cling. 2 | """ 3 | 4 | from typing import Tuple, List 5 | 6 | from hpccm.templates.git import git 7 | from hpccm.templates.CMakeBuild import CMakeBuild 8 | 9 | import xcc.config 10 | 11 | 12 | def build_cling( 13 | cling_url: str, 14 | config: xcc.config.XCC_Config, 15 | cling_branch=None, 16 | cling_hash=None, 17 | git_cling_opts=["--depth=1"], 18 | ) -> List[str]: 19 | """Return Cling build instructions. 20 | 21 | :param cling_url: GitHub url of the Cling repository 22 | :type cling_url: str 23 | :param cling_branch: GitHub branch of the Cling repository 24 | :type cling_branch: str 25 | :param cling_hash: GitHub commit hash of the Cling repository 26 | :type cling_hash: str 27 | :param config: Configuration object, which contains different information for the stage 28 | :type config: xcc.config.XCC_Config 29 | :param git_cling_opts: Setting options for Git Clone 30 | :type git_cling_opts: [str] 31 | :returns: a list of build instructions and a list of the install folders 32 | :rtype: [str],[str] 33 | 34 | """ 35 | compiler_threads = config.get_cmake_compiler_threads() 36 | linker_threads = config.get_cmake_linker_threads() 37 | 38 | cbc: List[str] = [] 39 | 40 | cbc += [ 41 | "", 42 | "#///////////////////////////////////////////////////////////", 43 | "#// Install Cling //", 44 | "#///////////////////////////////////////////////////////////", 45 | ] 46 | 47 | cbc += [ 48 | "", 49 | "#/////////////////////////////", 50 | "#// Download Cling sources //", 51 | "#/////////////////////////////", 52 | ] 53 | 54 | git_llvm = git() 55 | cbc.append( 56 | git_llvm.clone_step( 57 | repository="http://root.cern.ch/git/llvm.git", 58 | branch="cling-patches", 59 | path=config.build_prefix, 60 | directory="llvm", 61 | ) 62 | ) 63 | git_clang = git() 64 | cbc.append( 65 | git_clang.clone_step( 66 | repository="http://root.cern.ch/git/clang.git", 67 | branch="cling-patches", 68 | path=config.build_prefix + "/llvm/tools", 69 | ) 70 | ) 71 | git_cling = git(opts=git_cling_opts) 72 | cbc.append( 73 | git_cling.clone_step( 74 | repository=cling_url, 75 | branch=cling_branch, 76 | commit=cling_hash, 77 | path=config.build_prefix + "/llvm/tools", 78 | ) 79 | ) 80 | # add libc++ and libcxxabi to the llvm project 81 | # Comaker detect the projects automatically and builds it. 82 | if config.build_libcxx: 83 | git_libcxx = git() 84 | cbc.append( 85 | git_libcxx.clone_step( 86 | repository="https://github.com/llvm-mirror/libcxx", 87 | branch="release_50", 88 | path=config.build_prefix + "/llvm/projects", 89 | ) 90 | ) 91 | git_libcxxabi = git() 92 | cbc.append( 93 | git_libcxx.clone_step( 94 | repository="https://github.com/llvm-mirror/libcxxabi", 95 | branch="release_50", 96 | path=config.build_prefix + "/llvm/projects", 97 | ) 98 | ) 99 | 100 | for build in config.get_cling_build(): 101 | cbc += [ 102 | "", 103 | "#/////////////////////////////", 104 | "{:<28}".format("#// Build Cling " + build.build_type) + "//", 105 | "#/////////////////////////////", 106 | ] 107 | 108 | cmake_opts = [ 109 | "-G Ninja", 110 | "-DCMAKE_BUILD_TYPE=" + build.build_type, 111 | '-DLLVM_ABI_BREAKING_CHECKS="FORCE_OFF"', 112 | "-DCMAKE_LINKER=/usr/bin/gold", 113 | "-DLLVM_ENABLE_RTTI=ON", 114 | "'-DCMAKE_JOB_POOLS:STRING=compile={0};link={1}'".format( 115 | compiler_threads, linker_threads 116 | ), 117 | "'-DCMAKE_JOB_POOL_COMPILE:STRING=compile'", 118 | "'-DCMAKE_JOB_POOL_LINK:STRING=link'", 119 | '-DLLVM_TARGETS_TO_BUILD="host;NVPTX"', 120 | "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", 121 | ] 122 | 123 | # build the project with libc++ 124 | # the flag is not necessary to enable the build of libc++ and libc++abi 125 | if config.build_libcxx: 126 | cmake_opts.append("-DLLVM_ENABLE_LIBCXX=ON") 127 | 128 | cm_cling = CMakeBuild(prefix=build.install_path) 129 | cbc.append( 130 | cm_cling.configure_step( 131 | build_directory=build.build_path, 132 | directory=config.build_prefix + "/llvm", 133 | opts=cmake_opts, 134 | ) 135 | ) 136 | cbc.append(cm_cling.build_step(parallel=None, target="install")) 137 | 138 | cbc.append("PATH_bak=$PATH") 139 | cbc.append("PATH=$PATH:" + build.install_path + "/bin") 140 | cbc.append("cd " + build.install_path + "/share/cling/Jupyter/kernel") 141 | cbc.append(config.get_miniconda_path() + "/bin/pip install -e .") 142 | cbc.append("PATH=$PATH_bak") 143 | cbc.append("cd - ") 144 | 145 | if not config.keep_build: 146 | for build in config.get_cling_build(): 147 | config.paths_to_delete.append(build.build_path) 148 | config.paths_to_delete.append(config.build_prefix + "/llvm") 149 | 150 | return cbc 151 | -------------------------------------------------------------------------------- /xcc/config.py: -------------------------------------------------------------------------------- 1 | """The config object provides information for the different recipe generator stages. 2 | 3 | """ 4 | 5 | from typing import List, Union 6 | from copy import deepcopy 7 | 8 | supported_clang_version = [8, 9] 9 | 10 | 11 | class XCC_Config: 12 | class build_object: 13 | def __init__( 14 | self, 15 | build_path: str, 16 | install_path: str, 17 | build_type: str, 18 | cling_install_path: str = "", 19 | ): 20 | """Contains build path, install path, build type and related cling install path for a cling or xeus-cling build. 21 | 22 | :param build_path: Path, where cmake build the project. 23 | :type build_path: str 24 | :param install_path: Path, where cmake install the project. 25 | :type install_path: str 26 | :param build_type: CMAKE_BUILD_TYP 27 | :type build_type: str 28 | :param cling_install_path: Installation path of a cling build. 29 | :type cling_install_path: str 30 | 31 | """ 32 | self.build_path = build_path 33 | self.install_path = install_path 34 | self.build_type = build_type 35 | self.cling_install_path = cling_install_path 36 | 37 | def __str__(self): 38 | s = "build_path: " + self.build_path + "\n" 39 | s += "install_path: " + self.install_path + "\n" 40 | s += "build_type: " + self.build_type + "\n" 41 | s += "cling_install_path: " + self.cling_install_path + "\n" 42 | return s 43 | 44 | def __init__( 45 | self, 46 | container: str = "singularity", 47 | build_prefix: str = "/tmp", 48 | install_prefix: str = "/usr/local", 49 | build_type: str = "RELEASE", 50 | second_build_type: str = "", 51 | keep_build: bool = False, 52 | compiler_threads: int = 1, 53 | linker_threads: int = 1, 54 | build_libcxx: bool = False, 55 | clang_version: int = 8, 56 | gen_args: str = "", 57 | ): 58 | """Setup the configuration object 59 | 60 | :param container: 'docker' or 'singularity' 61 | :type container: str 62 | :param build_prefix: Prefix path of the source and build folders of the libraries and projects. 63 | :type build_prefix: str 64 | :param install_prefix: Prefix path where all projects will be installed. 65 | :type install_prefix: str 66 | :param build_type: Set CMAKE_BUILD_TYPE : 'DEBUG', 'RELEASE', 'RELWITHDEBINFO', 'MINSIZEREL' 67 | :type build_type: str 68 | :param second_build_type: If you want to build cling and xeus-cling a second time with different CMake build type. Set the CMake build type, for example RELEASE. 69 | :type second_build_type: str 70 | :param keep_build: Keep source and build files after installation. 71 | :type keep_build: str 72 | :param compiler_threads: Number of build threads for make (0 for all available threads). 73 | :type compiler_threads: int 74 | :param linker_threads: Number of linker threads for ninja (if 0, same number like threads) 75 | :type linker_threads: int 76 | :param build_libcxx: Build the whole stack with libc++. Also add the libc++ and libc++abi projects to the llvm build. 77 | :type build_libcxx: bool 78 | :param clang_version: Version of the project clang compiler (see XCC_Config.supported_clang_version) 79 | :type clang_version: int 80 | :param gen_args: The string will be save in the environment variable XCC_GEN_ARGS should be used the save the arguments of the generator script if None, no environment variable is created. 81 | :type gen_args: str 82 | 83 | """ 84 | self.author = "Simeon Ehrig" 85 | self.email = "s.ehrig@hzdr.de" 86 | self.version = "2.3" 87 | 88 | if clang_version not in supported_clang_version: 89 | raise ValueError( 90 | "Clang version " 91 | + str(clang_version) 92 | + " is not supported\n" 93 | + "Supported versions: " 94 | + ", ".join(map(str, supported_clang_version)) 95 | ) 96 | else: 97 | self.clang_version = clang_version 98 | 99 | if not check_build_type(build_type): 100 | raise ValueError( 101 | "build_type have to be: 'DEBUG', 'RELEASE', 'RELWITHDEBINFO', 'MINSIZEREL'" 102 | ) 103 | 104 | if second_build_type and not check_build_type(second_build_type): 105 | raise ValueError( 106 | "second_build_type have to be: 'DEBUG', 'RELEASE', 'RELWITHDEBINFO', 'MINSIZEREL'" 107 | ) 108 | 109 | self.container: str = container 110 | self.build_prefix: str = build_prefix 111 | self.install_prefix: str = install_prefix 112 | self.build_type: str = build_type 113 | self.second_build_type: str = second_build_type 114 | self.keep_build: bool = keep_build 115 | self.paths_to_delete: List[str] = [] 116 | self.compiler_threads: int = compiler_threads 117 | self.linker_threads: int = linker_threads 118 | self.build_libcxx: bool = build_libcxx 119 | self.gen_args: str = gen_args 120 | 121 | def get_copy(self): 122 | """Returns a deepcopy. 123 | 124 | :returns: Config object 125 | :rtype: XCC_Config 126 | 127 | """ 128 | c = XCC_Config( 129 | container=self.container, 130 | build_prefix=self.build_prefix, 131 | install_prefix=self.install_prefix, 132 | build_type=self.build_type, 133 | second_build_type=self.second_build_type, 134 | keep_build=self.keep_build, 135 | compiler_threads=self.compiler_threads, 136 | linker_threads=self.linker_threads, 137 | build_libcxx=self.build_libcxx, 138 | gen_args=self.gen_args, 139 | ) 140 | c.paths_to_delete = deepcopy(self.paths_to_delete) 141 | 142 | return c 143 | 144 | def get_cmake_compiler_threads(self) -> str: 145 | """Return a number or $(nproc), of compiler_threads is 0 146 | 147 | :returns: number of threads 148 | :rtype: str 149 | 150 | """ 151 | if self.compiler_threads is None: 152 | self.compiler_threads = 0 153 | return "$(nproc)" if self.compiler_threads == 0 else str(self.compiler_threads) 154 | 155 | def get_cmake_linker_threads(self) -> str: 156 | """Return a number or $(nproc), of linker_threads is 0 157 | 158 | :returns: number of threads 159 | :rtype: str 160 | 161 | """ 162 | 163 | if self.linker_threads is None: 164 | self.linker_threads = 0 165 | return ( 166 | str(self.compiler_threads) 167 | if self.linker_threads == 0 168 | else str(self.linker_threads) 169 | ) 170 | 171 | def get_cling_build(self) -> List[build_object]: 172 | """Create a list of build configurations for cling. 173 | 174 | :returns: build configurations 175 | :rtype: List[build_object] 176 | 177 | """ 178 | cling_builds: List[XCC_Config.build_object] = [] 179 | 180 | if not self.second_build_type: 181 | cling_builds.append( 182 | XCC_Config.build_object( 183 | build_path=self.build_prefix + "/cling_build", 184 | install_path=self.install_prefix, 185 | build_type=self.build_type, 186 | ) 187 | ) 188 | else: 189 | cling_builds.append( 190 | XCC_Config.build_object( 191 | build_path=self.build_prefix + "/build_" + self.build_type.lower(), 192 | install_path=self.install_prefix 193 | + "/install_" 194 | + self.build_type.lower(), 195 | build_type=self.build_type, 196 | ) 197 | ) 198 | cling_builds.append( 199 | XCC_Config.build_object( 200 | build_path=self.build_prefix 201 | + "/build_" 202 | + self.second_build_type.lower(), 203 | install_path=self.install_prefix 204 | + "/install_" 205 | + self.second_build_type.lower(), 206 | build_type=self.second_build_type, 207 | ) 208 | ) 209 | 210 | return cling_builds 211 | 212 | def get_xeus_cling_build(self) -> List[build_object]: 213 | """Create a list of build configurations for xeus-cling. 214 | 215 | :returns: build configurations 216 | :rtype: List[build_object] 217 | 218 | """ 219 | xeus_cling_builds: List[XCC_Config.build_object] = [] 220 | 221 | if not self.second_build_type: 222 | xeus_cling_builds.append( 223 | XCC_Config.build_object( 224 | build_path=self.build_prefix + "/xeus-cling_build", 225 | install_path="", 226 | build_type=self.build_type, 227 | cling_install_path=self.install_prefix, 228 | ) 229 | ) 230 | else: 231 | xeus_cling_builds.append( 232 | XCC_Config.build_object( 233 | build_path=self.build_prefix 234 | + "/xeus-cling_build_" 235 | + self.build_type.lower(), 236 | install_path="", 237 | build_type=self.build_type, 238 | cling_install_path=self.install_prefix 239 | + "/install_" 240 | + self.build_type.lower(), 241 | ) 242 | ) 243 | xeus_cling_builds.append( 244 | XCC_Config.build_object( 245 | build_path=self.build_prefix 246 | + "/xeus-cling_build_" 247 | + self.second_build_type.lower(), 248 | install_path="", 249 | build_type=self.second_build_type, 250 | cling_install_path=self.install_prefix 251 | + "/install_" 252 | + self.second_build_type.lower(), 253 | ) 254 | ) 255 | 256 | return xeus_cling_builds 257 | 258 | def get_miniconda_path(self) -> str: 259 | """Create the miniconda install path 260 | 261 | :returns: returns the path to miniconda 262 | :rtype: str 263 | 264 | """ 265 | return self.install_prefix + "/miniconda3" 266 | 267 | 268 | def check_build_type(type: str) -> bool: 269 | """Check if input is a CMAKE_BUILD_TYPE: 'DEBUG', 'RELEASE', 'RELWITHDEBINFO', 'MINSIZEREL' 270 | 271 | :param type: type to check 272 | :type type: str 273 | :returns: true if it supported type, otherwise false 274 | :rtype: bool 275 | 276 | """ 277 | if type in ["DEBUG", "RELEASE", "RELWITHDEBINFO", "MINSIZEREL"]: 278 | return True 279 | else: 280 | return False 281 | -------------------------------------------------------------------------------- /xcc/generator.py: -------------------------------------------------------------------------------- 1 | """The file contains a generator class that can create different versions of the 2 | xeus-cling-cuda stack. The class has no user interface and is designed for 3 | integration into other projects. Use the 4 | 5 | * gen_devel_stage() 6 | * gen_release_single_stage() 7 | 8 | """ 9 | 10 | from typing import Tuple, List, Dict, Union 11 | from copy import deepcopy 12 | import hpccm 13 | from hpccm.primitives import baseimage, shell, environment, raw, copy, runscript, label 14 | from hpccm.building_blocks.packages import packages 15 | from hpccm.building_blocks.cmake import cmake 16 | from hpccm.building_blocks.llvm import llvm 17 | from hpccm.templates.rm import rm 18 | 19 | from xcc.cling import build_cling 20 | from xcc.xeuscling import build_xeus_cling 21 | from xcc.helper import build_git_and_cmake, add_libcxx_cmake_arg 22 | from xcc.openssl import build_openssl 23 | from xcc.miniconda import build_miniconda 24 | from xcc.jupyter import build_dev_jupyter_kernel, build_rel_jupyter_kernel 25 | from xcc.basestage import gen_base_stage 26 | import xcc.config 27 | 28 | 29 | class XCC_gen: 30 | def __init__( 31 | self, 32 | container="singularity", 33 | build_prefix="/tmp", 34 | install_prefix="/usr/local", 35 | build_type="RELEASE", 36 | keep_build=False, 37 | threads=None, 38 | linker_threads=None, 39 | clang_version=8, 40 | gen_args=None, 41 | build_libcxx=None, 42 | ): 43 | """Set up the basic configuration of all projects in the container. There are only a few exceptions in the dev-stage, see gen_devel_stage(). 44 | 45 | :param container: 'docker' or 'singularity' 46 | :type container: str 47 | :param build_prefix: Prefix path of the source and build folders of the libraries and projects. 48 | :type build_prefix: str 49 | :param install_prefix: Prefix path where all projects will be installed. 50 | :type install_prefix: str 51 | :param build_type: Set CMAKE_BUILD_TYPE : 'DEBUG', 'RELEASE', 'RELWITHDEBINFO', 'MINSIZEREL' 52 | :type build_type: str 53 | :param keep_build: Keep source and build files after installation. 54 | :type keep_build: str 55 | :param threads: number of build threads for make (None for all available threads) 56 | :type threads: int 57 | :param linker_threads: number of linker threads for ninja (if None, same number like threads) 58 | :type linker_threads: int 59 | :param clang_version: version of the project clang compiler (default: 8 - supported: 8, 9) 60 | :type clang_version: int 61 | :param gen_args: The string will be save in the environment variable XCC_GEN_ARGS should be used the save the arguments of the generator script if None, no environment variable is created. 62 | :type gen_args: str 63 | :param build_libcxx: Build the whole stack with libc++. Also add the libc++ and libc++abi projects to the llvm build. 64 | :type build_libcxx: bool 65 | 66 | """ 67 | self.config = xcc.config.XCC_Config( 68 | container=container, 69 | build_prefix=build_prefix, 70 | install_prefix=install_prefix, 71 | build_type=build_type, 72 | keep_build=keep_build, 73 | compiler_threads=threads, 74 | linker_threads=linker_threads, 75 | build_libcxx=bool(build_libcxx), 76 | clang_version=clang_version, 77 | gen_args=gen_args, 78 | ) 79 | 80 | # the list contains all projects with properties that are built and 81 | # installed from source code 82 | # the list contains dictionaries with at least two entries: name and tag 83 | # * name is a unique identifier 84 | # * tag describes which build function must be used 85 | # the order of the list is important for the build steps 86 | self.project_list = [] # type: ignore 87 | 88 | self.cling_url = "https://github.com/root-project/cling.git" 89 | self.cling_branch = None 90 | self.cling_hash = "595580b" 91 | 92 | # have to be before building cling, because the cling jupyter kernel 93 | # needs pip 94 | self.project_list.append({"name": "miniconda3", "tag": "miniconda"}) 95 | 96 | self.project_list.append({"name": "cling", "tag": "cling"}) 97 | 98 | ####################################################################### 99 | # xeus dependencies 100 | ####################################################################### 101 | self.project_list.append({"name": "openssl", "tag": "openssl"}) 102 | 103 | self.add_git_cmake_entry( 104 | name="libzmq", 105 | url="https://github.com/zeromq/libzmq.git", 106 | branch="v4.2.5", 107 | opts=[ 108 | "-DWITH_PERF_TOOL=OFF", 109 | "-DZMQ_BUILD_TESTS=OFF", 110 | "-DENABLE_CPACK=OFF", 111 | "-DCMAKE_BUILD_TYPE=" + build_type, 112 | ], 113 | ) 114 | self.add_git_cmake_entry( 115 | name="cppzmq", 116 | url="https://github.com/zeromq/cppzmq.git", 117 | branch="v4.3.0", 118 | opts=["-DCMAKE_BUILD_TYPE=" + build_type], 119 | ) 120 | self.add_git_cmake_entry( 121 | name="nlohmann_json", 122 | url="https://github.com/nlohmann/json.git", 123 | branch="v3.7.0", 124 | opts=["-DCMAKE_BUILD_TYPE=" + build_type], 125 | ) 126 | self.add_git_cmake_entry( 127 | name="xtl", 128 | url="https://github.com/QuantStack/xtl.git", 129 | branch="0.6.9", 130 | opts=["-DCMAKE_BUILD_TYPE=" + build_type], 131 | ) 132 | self.add_git_cmake_entry( 133 | name="xeus", 134 | url="https://github.com/QuantStack/xeus.git", 135 | branch="0.23.3", 136 | opts=[ 137 | "-DBUILD_EXAMPLES=OFF", 138 | "-DDISABLE_ARCH_NATIVE=ON", 139 | "-DCMAKE_BUILD_TYPE=" + build_type, 140 | ], 141 | ) 142 | 143 | ####################################################################### 144 | ### xeus-cling and dependencies 145 | ####################################################################### 146 | self.add_git_cmake_entry( 147 | name="pugixml", 148 | url="https://github.com/zeux/pugixml.git", 149 | branch="v1.8.1", 150 | opts=[ 151 | "-DCMAKE_BUILD_TYPE=" + build_type, 152 | "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", 153 | ], 154 | ) 155 | self.add_git_cmake_entry( 156 | name="cxxopts", 157 | url="https://github.com/jarro2783/cxxopts.git", 158 | branch="v2.2.0", 159 | opts=["-DCMAKE_BUILD_TYPE=" + build_type], 160 | ) 161 | self.project_list.append( 162 | { 163 | "name": "xeus-cling", 164 | "tag": "xeus-cling", 165 | "url": "https://github.com/QuantStack/xeus-cling.git", 166 | "branch": "0.8.0", 167 | } 168 | ) 169 | 170 | self.project_list.append({"name": "jupyter_kernel", "tag": "jupyter_kernel"}) 171 | 172 | self.add_git_cmake_entry( 173 | name="xproperty", 174 | url="https://github.com/QuantStack/xproperty.git", 175 | branch="0.8.1", 176 | opts=["-DCMAKE_BUILD_TYPE=" + build_type], 177 | ) 178 | 179 | self.add_git_cmake_entry( 180 | name="xwidgets", 181 | url="https://github.com/QuantStack/xwidgets.git", 182 | branch="0.19.0", 183 | opts=["-DCMAKE_BUILD_TYPE=" + build_type], 184 | ) 185 | 186 | def add_git_cmake_entry( 187 | self, name: str, url: str, branch: str, opts: List[str] = [] 188 | ): 189 | """add git-and-cmake entry to self.project_list. 190 | 191 | The shape is: 192 | {'name' : name, 193 | 'url' : url, 194 | 'branch' : branch, 195 | 'opts' : opts} 196 | 197 | :param name: name of the project 198 | :type name: str 199 | :param url: git clone url 200 | :type url: str 201 | :param branch: branch or version (git clone --branch) 202 | :type branch: str 203 | :param opts: a list of CMAKE arguments (e.g. -DCMAKE_BUILD_TYPE=RELEASE) 204 | :type opts: [str] 205 | """ 206 | if self.config.build_libcxx: 207 | opts = add_libcxx_cmake_arg(opts) 208 | 209 | self.project_list.append( 210 | { 211 | "name": name, 212 | "tag": "git_cmake", 213 | "url": url, 214 | "branch": branch, 215 | "opts": opts, 216 | } 217 | ) 218 | 219 | def gen_devel_stage( 220 | self, project_path: str, dual_build_type: str = "" 221 | ) -> hpccm.Stage: 222 | """Get a recipe for the development stack. The build process is divided into two parts. The first is building the container. The container contains all software parts that should not be changed during development. The second part contains a runscript that downloads and build the software parts that can be modified, e.g. cling. 223 | 224 | :param project_path: Path on the host system on which the modifiable software projects live 225 | :type project_path: str 226 | :param dual_build_type: If you want to build cling and xeus-cling a second time with different CMake build type. Set the CMake build type, for example RELEASE 227 | :type dual_build_type: str 228 | :returns: hpccm Stage 229 | :rtype: hpccm.Stage 230 | 231 | """ 232 | stage0 = gen_base_stage(self.config) 233 | # set the path to the changeable project as environment variable 234 | stage0 += environment(variables={"XCC_PROJECT_PATH": project_path}) 235 | 236 | # the following projects are being built outside the container 237 | self.__gen_project_builds( 238 | stage=stage0, 239 | exclude_list=["cling", "xeus-cling", "miniconda", "jupyter_kernel"], 240 | ) 241 | 242 | if not self.config.keep_build: 243 | r = rm() 244 | stage0 += shell( 245 | commands=[r.cleanup_step(items=self.config.paths_to_delete)] 246 | ) 247 | 248 | stage0 += raw(docker="EXPOSE 8888") 249 | 250 | runscript_config = self.config.get_copy() 251 | runscript_config.build_prefix = project_path 252 | runscript_config.install_prefix = project_path 253 | runscript_config.second_build_type = dual_build_type 254 | runscript_config.keep_build = True 255 | 256 | cm_runscript: List[str] = [] 257 | # set clang as compiler 258 | cm_runscript += [ 259 | "export CC=clang-" + str(self.config.clang_version), 260 | "export CXX=clang++-" + str(self.config.clang_version), 261 | ] 262 | 263 | ################################################################## 264 | # miniconda 265 | ################################################################## 266 | cm, env = build_miniconda(config=runscript_config) 267 | stage0 += environment(variables=env) 268 | cm_runscript += cm 269 | 270 | ################################################################## 271 | # cling 272 | ################################################################## 273 | # the default behavior is PREFIX=/usr/local/ -> install to /usr/local/bin ... 274 | # for development it is better to install to project_path/install 275 | # if the second build is activated, two different installation folders will be created automatically 276 | cm = build_cling( 277 | cling_url=self.cling_url, 278 | cling_branch=self.cling_branch, 279 | cling_hash=self.cling_hash, 280 | config=runscript_config, 281 | git_cling_opts=[""], 282 | ) 283 | cm_runscript += cm 284 | 285 | ################################################################## 286 | # xeus-cling 287 | ################################################################## 288 | for p in self.project_list: 289 | if p["name"] == "xeus-cling": 290 | xc = p 291 | 292 | cm_runscript += build_xeus_cling( 293 | url=xc["url"], branch=xc["branch"], config=runscript_config, 294 | ) 295 | 296 | cm_runscript += build_dev_jupyter_kernel(config=runscript_config) 297 | 298 | stage0 += runscript(commands=cm_runscript) 299 | return stage0 300 | 301 | def gen_release_single_stage(self) -> hpccm.Stage: 302 | """Get a release recipe for the stack. The stack contains a single stage. Requires a little more memory on singularity and much on docker, but it is less error prone. 303 | 304 | :returns: hpccm Stage 305 | :rtype: hpccm.Stage 306 | 307 | """ 308 | stage0 = gen_base_stage(self.config) 309 | 310 | self.__gen_project_builds(stage=stage0) 311 | 312 | if not self.config.keep_build: 313 | r = rm() 314 | stage0 += shell( 315 | commands=[r.cleanup_step(items=self.config.paths_to_delete)] 316 | ) 317 | 318 | stage0 += raw(docker="EXPOSE 8888") 319 | 320 | return stage0 321 | 322 | def __gen_project_builds(self, stage: hpccm.Stage, exclude_list=[]): 323 | """Add build instructions to the stage of the various projects contained in self.project_list 324 | 325 | :param stage: hpccm stage in which the instructions are added 326 | :type stage: hpccm.Stage 327 | :param exclude_list: List of names, which will skipped. Can be used when a project is added otherwise. 328 | :type exclude_list: [str] 329 | 330 | """ 331 | for p in self.project_list: 332 | if p["tag"] == "cling": 333 | if "cling" not in exclude_list: 334 | stage += shell( 335 | commands=build_cling( 336 | cling_url=self.cling_url, 337 | cling_branch=self.cling_branch, 338 | cling_hash=self.cling_hash, 339 | config=self.config, 340 | ) 341 | ) 342 | elif p["tag"] == "xeus-cling": 343 | if "xeus-cling" not in exclude_list: 344 | stage += shell( 345 | commands=build_xeus_cling( 346 | url=p["url"], branch=p["branch"], config=self.config, 347 | ) 348 | ) 349 | elif p["tag"] == "git_cmake": 350 | if p["name"] not in exclude_list: 351 | stage += shell( 352 | commands=build_git_and_cmake( 353 | name=p["name"], 354 | url=p["url"], 355 | branch=p["branch"], 356 | config=self.config, 357 | opts=p["opts"], 358 | ) 359 | ) 360 | elif p["tag"] == "openssl": 361 | if "openssl" not in exclude_list: 362 | shc, env = build_openssl(name="openssl-1.1.1c", config=self.config,) 363 | stage += shell(commands=shc) 364 | stage += environment(variables=env) 365 | elif p["tag"] == "miniconda": 366 | if "miniconda" not in exclude_list: 367 | shc, env = build_miniconda(config=self.config,) 368 | stage += shell(commands=shc) 369 | stage += environment(variables=env) 370 | elif p["tag"] == "jupyter_kernel": 371 | if "jupyter_kernel" not in exclude_list: 372 | stage += shell( 373 | commands=build_rel_jupyter_kernel(config=self.config,) 374 | ) 375 | else: 376 | raise ValueError("unknown tag: " + p["tag"]) 377 | 378 | def __str__(self): 379 | s = "" 380 | for p in self.project_list: 381 | if p["tag"] == "git_cmake": 382 | s = s + "Git and CMake: " + p["name"] + "\n" 383 | else: 384 | s = s + p["tag"] + ": " + p["name"] + "\n" 385 | 386 | return s 387 | -------------------------------------------------------------------------------- /xcc/helper.py: -------------------------------------------------------------------------------- 1 | """Different helper functions for generating container recipe 2 | """ 3 | 4 | from typing import List, Union 5 | 6 | from hpccm.templates.git import git 7 | from hpccm.templates.CMakeBuild import CMakeBuild 8 | 9 | import xcc.config 10 | 11 | 12 | def build_git_and_cmake( 13 | name: str, 14 | url: str, 15 | branch: str, 16 | config: xcc.config.XCC_Config, 17 | opts=[], 18 | ) -> List[str]: 19 | """Combines git clone, cmake and cmake traget='install' 20 | 21 | :param name: name of the project 22 | :type name: str 23 | :param url: git clone url 24 | :type url: str 25 | :param branch: branch or version (git clone --branch) 26 | :type branch: str 27 | :param config: Configuration object, which contains different information for the stage 28 | :type config: xcc.config.XCC_Config 29 | :param opts: a list of CMAKE arguments (e.g. -DCMAKE_BUILD_TYPE=RELEASE) 30 | :type opts: List[str] 31 | :returns: list of bash commands for git and cmake 32 | :rtype: List[str] 33 | 34 | """ 35 | # commands 36 | cm = [ 37 | "", 38 | "#///////////////////////////////////////////////////////////", 39 | "{:<58}".format("#// Build " + name) + "//", 40 | "#///////////////////////////////////////////////////////////", 41 | ] 42 | 43 | git_conf = git() 44 | cm.append( 45 | git_conf.clone_step( 46 | repository=url, branch=branch, path=config.build_prefix, directory=name 47 | ) 48 | ) 49 | cmake_conf = CMakeBuild(prefix=config.install_prefix) 50 | cm_build_dir = config.build_prefix + "/" + name + "_build" 51 | cm_source_dir = config.build_prefix + "/" + name 52 | cm.append( 53 | cmake_conf.configure_step( 54 | build_directory=cm_build_dir, directory=cm_source_dir, opts=opts 55 | ) 56 | ) 57 | cm.append(cmake_conf.build_step(parallel=config.get_cmake_compiler_threads(), target="install")) 58 | if not config.keep_build: 59 | config.paths_to_delete.append(cm_build_dir) 60 | config.paths_to_delete.append(cm_source_dir) 61 | return cm 62 | 63 | def add_libcxx_cmake_arg(inputList: List[str]) -> List[str]: 64 | """If the class attribute build_libcxx is true, add -DCMAKE_CXX_FLAGS="-stdlib=libc++" to cmake flags in inputlist. 65 | 66 | :param inputlist: List of cmake flags 67 | :type inputlist: List[str] 68 | :returns: inputlist plus -DCMAKE_CXX_FLAGS="-stdlib=libc++" if self.build_libcxx is true 69 | :rtype: List[str] 70 | 71 | """ 72 | for i, elem in enumerate(inputList): 73 | if elem.startswith('-DCMAKE_CXX_FLAGS="'): 74 | inputList[i] = elem[:-1] + ' -stdlib=libc++"' 75 | return inputList 76 | 77 | inputList.append('-DCMAKE_CXX_FLAGS="-stdlib=libc++"') 78 | return inputList 79 | -------------------------------------------------------------------------------- /xcc/jupyter.py: -------------------------------------------------------------------------------- 1 | """Functions to create build instructions for jupyter notebook and kernels. 2 | """ 3 | 4 | from typing import Dict, List, Union 5 | import json 6 | 7 | import xcc.config 8 | 9 | 10 | def build_rel_jupyter_kernel( 11 | config: xcc.config.XCC_Config, user_install=False, 12 | ) -> List[str]: 13 | """Returns jupyter kernel and instructions to install it 14 | 15 | :param config: Configuration object, which contains different information for the stage 16 | :type config: xcc.config.XCC_Config 17 | :param user_install: if true, add flag --user to jupyter-kernelspec install 18 | :type user_install: bool 19 | :returns: list of bash commands 20 | :rtype: List[str] 21 | 22 | """ 23 | user_install_arg = "" 24 | if user_install: 25 | user_install_arg = "--user " 26 | 27 | kernel_register: List[str] = [] 28 | # xeus-cling cuda kernel 29 | for std in [11, 14, 17]: 30 | kernel_register += [ 31 | "", 32 | "#/////////////////////////////", 33 | "{:<28}".format("#// Jupyter Kernel: Xeus " + str(std) + " cuda") + "//", 34 | "#/////////////////////////////", 35 | ] 36 | kernel_path = config.build_prefix + "/xeus-cling-cpp" + str(std) + "-cuda" 37 | kernel_register.append("mkdir -p " + kernel_path) 38 | kernel_register.append( 39 | "echo '" 40 | + gen_xeus_cling_jupyter_kernel(config.get_miniconda_path(), std) 41 | + "' > " 42 | + kernel_path 43 | + "/kernel.json" 44 | ) 45 | kernel_register.append( 46 | "jupyter-kernelspec install " + user_install_arg + kernel_path 47 | ) 48 | if not config.keep_build: 49 | config.paths_to_delete.append(kernel_path) 50 | 51 | # cling-cpp kernel 52 | for std in [11, 14, 17]: 53 | kernel_register += [ 54 | "", 55 | "#/////////////////////////////", 56 | "{:<28}".format("#// Jupyter Kernel: Cling " + str(std)) + "//", 57 | "#/////////////////////////////", 58 | ] 59 | kernel_path = config.build_prefix + "/cling-cpp" + str(std) 60 | kernel_register.append("mkdir -p " + kernel_path) 61 | kernel_register.append( 62 | "echo '" 63 | + gen_cling_jupyter_kernel(std, False) 64 | + "' > " 65 | + kernel_path 66 | + "/kernel.json" 67 | ) 68 | kernel_register.append( 69 | "jupyter-kernelspec install " + user_install_arg + kernel_path 70 | ) 71 | if not config.keep_build: 72 | config.paths_to_delete.append(kernel_path) 73 | 74 | # cling-cuda kernel 75 | for std in [11, 14, 17]: 76 | kernel_register += [ 77 | "", 78 | "#/////////////////////////////", 79 | "{:<28}".format("#// Jupyter Kernel: Cling " + str(std) + " cuda") + "//", 80 | "#/////////////////////////////", 81 | ] 82 | kernel_path = config.build_prefix + "/cling-cpp" + str(std) + "-cuda" 83 | kernel_register.append("mkdir -p " + kernel_path) 84 | kernel_register.append( 85 | "echo '" 86 | + gen_cling_jupyter_kernel(std, True) 87 | + "' > " 88 | + kernel_path 89 | + "/kernel.json" 90 | ) 91 | kernel_register.append( 92 | "jupyter-kernelspec install " + user_install_arg + kernel_path 93 | ) 94 | if not config.keep_build: 95 | config.paths_to_delete.append(kernel_path) 96 | 97 | return kernel_register 98 | 99 | 100 | def build_dev_jupyter_kernel(config: xcc.config.XCC_Config) -> List[str]: 101 | """Returns jupyter kernel and instructions to install it in the miniconda3 folder. For release builds, please use build_rel_jupyter_kernel(). 102 | 103 | :param config: Configuration object, which contains different information for the stage 104 | :type config: xcc.config.XCC_Config 105 | :returns: list of bash commands 106 | :rtype: List[str] 107 | 108 | """ 109 | kernel_prefix = config.build_prefix + "/kernels" 110 | 111 | kernel_register = [] 112 | kernel_register.append( 113 | "mkdir -p " + config.get_miniconda_path() + "/share/jupyter/kernels/" 114 | ) 115 | 116 | # xeus-cling cuda kernel 117 | for std in [11, 14, 17]: 118 | kernel_register += [ 119 | "", 120 | "#/////////////////////////////", 121 | "{:<28}".format("#// Jupyter Kernel: Xeus " + str(std) + " cuda") + "//", 122 | "#/////////////////////////////", 123 | ] 124 | kernel_path = kernel_prefix + "/xeus-cling-cpp" + str(std) + "-cuda" 125 | kernel_register.append("mkdir -p " + kernel_path) 126 | kernel_register.append( 127 | "echo '" 128 | + gen_xeus_cling_jupyter_kernel(config.get_miniconda_path(), std) 129 | + "' > " 130 | + kernel_path 131 | + "/kernel.json" 132 | ) 133 | kernel_register.append( 134 | "cp -r " 135 | + kernel_path 136 | + " " 137 | + config.get_miniconda_path() 138 | + "/share/jupyter/kernels/" 139 | ) 140 | if not config.keep_build: 141 | config.paths_to_delete.append(kernel_path) 142 | 143 | # cling-cpp kernel 144 | for std in [11, 14, 17]: 145 | kernel_register += [ 146 | "", 147 | "#/////////////////////////////", 148 | "{:<28}".format("#// Jupyter Kernel: Cling " + str(std)) + "//", 149 | "#/////////////////////////////", 150 | ] 151 | kernel_path = kernel_prefix + "/cling-cpp" + str(std) 152 | kernel_register.append("mkdir -p " + kernel_path) 153 | kernel_register.append( 154 | "echo '" 155 | + gen_cling_jupyter_kernel(std, False) 156 | + "' > " 157 | + kernel_path 158 | + "/kernel.json" 159 | ) 160 | kernel_register.append( 161 | "cp -r " 162 | + kernel_path 163 | + " " 164 | + config.get_miniconda_path() 165 | + "/share/jupyter/kernels/" 166 | ) 167 | if not config.keep_build: 168 | config.paths_to_delete.append(kernel_path) 169 | 170 | # cling-cuda kernel 171 | for std in [11, 14, 17]: 172 | kernel_register += [ 173 | "", 174 | "#/////////////////////////////", 175 | "{:<28}".format("#// Jupyter Kernel: Cling " + str(std) + " cuda") + "//", 176 | "#/////////////////////////////", 177 | ] 178 | kernel_path = kernel_prefix + "/cling-cpp" + str(std) + "-cuda" 179 | kernel_register.append("mkdir -p " + kernel_path) 180 | kernel_register.append( 181 | "echo '" 182 | + gen_cling_jupyter_kernel(std, True) 183 | + "' > " 184 | + kernel_path 185 | + "/kernel.json" 186 | ) 187 | kernel_register.append( 188 | "cp -r " 189 | + kernel_path 190 | + " " 191 | + config.get_miniconda_path() 192 | + "/share/jupyter/kernels/" 193 | ) 194 | if not config.keep_build: 195 | config.paths_to_delete.append(kernel_path) 196 | 197 | return kernel_register 198 | 199 | 200 | def gen_xeus_cling_jupyter_kernel(miniconda_path: str, cxx_std: int) -> str: 201 | """Generate jupyter kernel description files with cuda support for different C++ standards. The kernels uses xeus-cling. 202 | 203 | :param miniconda_prefix: path to the miniconda installation 204 | :type miniconda_prefix: str 205 | :param cxx_std: C++ Standard as number (options: 11, 14, 17) 206 | :type cxx_std: int 207 | :returns: json string 208 | :rtype: str 209 | 210 | """ 211 | return json.dumps( 212 | { 213 | "display_name": "Xeus-C++" + str(cxx_std) + "-CUDA", 214 | "argv": [ 215 | miniconda_path + "/bin/xcpp", 216 | "-f", 217 | "{connection_file}", 218 | "-std=c++" + str(cxx_std), 219 | "-xcuda", 220 | ], 221 | "language": "C++" + str(cxx_std), 222 | } 223 | ) 224 | 225 | 226 | def gen_cling_jupyter_kernel(cxx_std: int, cuda: bool) -> str: 227 | """Generate jupyter kernel description files with cuda support for different C++ standards. The kernels uses the jupyter kernel of the cling project. 228 | 229 | :param cxx_std: C++ Standard as number (options: 11, 14, 17) 230 | :type cxx_std: int 231 | :param cuda: if true, create kernel description file with cuda support 232 | :type cuda: bool 233 | :returns: json string 234 | :rtype: str 235 | 236 | """ 237 | kernel_json = { 238 | "display_name": "Cling-C++" + str(cxx_std) + ("-CUDA" if cuda else ""), 239 | "argv": [ 240 | "jupyter-cling-kernel", 241 | "-f", 242 | "{connection_file}", 243 | "--std=c++" + str(cxx_std), 244 | ], 245 | "language": "C++", 246 | } 247 | 248 | if cuda: 249 | kernel_json["env"] = {"CLING_OPTS": "-xcuda"} # type: ignore 250 | 251 | return json.dumps(kernel_json) 252 | -------------------------------------------------------------------------------- /xcc/miniconda.py: -------------------------------------------------------------------------------- 1 | """Function to create build instructions for miniconda. 2 | """ 3 | 4 | from typing import Tuple, List, Dict 5 | 6 | from hpccm.templates.rm import rm 7 | 8 | import xcc.config 9 | 10 | 11 | def build_miniconda(config: xcc.config.XCC_Config) -> Tuple[List[str], Dict[str, str]]: 12 | """Return Miniconda 3 installation instructions 13 | 14 | :param config: Configuration object, which contains different information for the stage 15 | :type config: xcc.config.XCC_Config 16 | :returns: list of bash commands and dictionary of environment variables 17 | :rtype: [str], {str,str} 18 | 19 | """ 20 | conda_bin = config.install_prefix + "/miniconda3/bin/" 21 | conda_exe = conda_bin + "conda" 22 | cm = [ 23 | "", 24 | "#///////////////////////////////////////////////////////////", 25 | "#// Install Miniconda 3 //", 26 | "#///////////////////////////////////////////////////////////", 27 | "cd /tmp", 28 | "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh", 29 | "chmod u+x Miniconda3-latest-Linux-x86_64.sh", 30 | "./Miniconda3-latest-Linux-x86_64.sh -b -p " 31 | + config.install_prefix 32 | + "/miniconda3", 33 | "export PATH=$PATH:" + conda_bin, 34 | conda_exe + " install -y -c conda-forge nodejs", 35 | conda_exe + " install -y jupyter", 36 | conda_exe + " install -y -c conda-forge jupyterlab", 37 | conda_exe + " install -y -c biobuilds libuuid", 38 | conda_exe + " install -y widgetsnbextension -c conda-forge", 39 | conda_bin + "jupyter labextension install @jupyter-widgets/jupyterlab-manager", 40 | "rm /tmp/Miniconda3-latest-Linux-x86_64.sh", 41 | "cd -", 42 | ] 43 | 44 | return cm, {"PATH": "$PATH:" + conda_bin} 45 | -------------------------------------------------------------------------------- /xcc/openssl.py: -------------------------------------------------------------------------------- 1 | """Function to create build instructions for openssl. 2 | """ 3 | 4 | from typing import Union, List, Tuple, List, Dict 5 | 6 | from hpccm.templates.wget import wget 7 | from hpccm.templates.tar import tar 8 | 9 | import xcc.config 10 | 11 | 12 | def build_openssl( 13 | name: str, config: xcc.config.XCC_Config, 14 | ) -> Tuple[List[str], Dict[str, str]]: 15 | """install openssl 16 | 17 | :param name: Name of the version (e.g. openssl-1.1.1c). Should be sliced from the official URL. 18 | :type name: str 19 | :param config: Configuration object, which contains different information for the stage 20 | :type config: xcc.config.XCC_Config 21 | :returns: list of bash commands and dictionary of environment variables 22 | :rtype: List[str], {str,str} 23 | 24 | """ 25 | make_threads = config.get_cmake_compiler_threads() 26 | 27 | cm = [ 28 | "", 29 | "#///////////////////////////////////////////////////////////", 30 | "#// Install OpenSSL //", 31 | "#///////////////////////////////////////////////////////////", 32 | ] 33 | wget_ssl = wget() 34 | tar_ssl = tar() 35 | cm.append( 36 | wget_ssl.download_step( 37 | url="https://www.openssl.org/source/" + name + ".tar.gz", 38 | directory=config.build_prefix, 39 | ) 40 | ) 41 | cm.append( 42 | tar_ssl.untar_step( 43 | tarball=config.build_prefix + "/" + name + ".tar.gz", 44 | directory=config.build_prefix, 45 | ) 46 | ) 47 | cm.append("cd " + config.build_prefix + "/" + name) 48 | cm.append( 49 | "./config --prefix=" + config.install_prefix + " -Wl,-rpath=/usr/local/lib" 50 | ) 51 | cm.append("make -j" + make_threads) 52 | cm.append("make install -j" + make_threads) 53 | cm.append("cd -") 54 | if not config.keep_build: 55 | config.paths_to_delete.append(config.build_prefix + "/" + name) 56 | config.paths_to_delete.append(config.build_prefix + "/" + name + ".tar.gz") 57 | 58 | return cm, {"OPENSSL_ROOT_DIR": config.install_prefix} 59 | -------------------------------------------------------------------------------- /xcc/xeuscling.py: -------------------------------------------------------------------------------- 1 | """Function to create xeus-cling build instruction. 2 | """ 3 | 4 | from typing import List, Union 5 | 6 | from hpccm.templates.git import git 7 | from hpccm.templates.CMakeBuild import CMakeBuild 8 | 9 | import xcc.config 10 | from xcc.helper import add_libcxx_cmake_arg 11 | 12 | 13 | def build_xeus_cling( 14 | url: str, branch: str, config: xcc.config.XCC_Config, 15 | ) -> List[str]: 16 | """Return Cling build instructions. 17 | 18 | :param url: git clone url 19 | :type url: str 20 | :param branch: branch or version (git clone --branch) 21 | :type branch: str 22 | :param config: Configuration object, which contains different information for the stage 23 | :type config: xcc.config.XCC_Config 24 | :returns: a list of build instructions 25 | :rtype: List[str] 26 | 27 | """ 28 | cm = [ 29 | "", 30 | "#///////////////////////////////////////////////////////////", 31 | "#// Install Xeus-Cling //", 32 | "#///////////////////////////////////////////////////////////", 33 | ] 34 | git_conf = git() 35 | cm.append( 36 | git_conf.clone_step( 37 | repository=url, 38 | branch=branch, 39 | path=config.build_prefix, 40 | directory="xeus-cling", 41 | ) 42 | ) 43 | 44 | # backup PATH 45 | # xeus-cling requires the llvm-config executable file from the cling installation 46 | # for dual build different bin paths are necessary 47 | cm.append("bPATH=$PATH") 48 | for build in config.get_xeus_cling_build(): 49 | cm += [ 50 | "", 51 | "#/////////////////////////////", 52 | "{:<28}".format("#// Build Xeus-Cling " + build.build_type) + "//", 53 | "#/////////////////////////////", 54 | ] 55 | # add path to llvm-config for the xeus-cling build 56 | cm.append("PATH=$bPATH:/" + build.cling_install_path + "/bin") 57 | 58 | cmake_opts = [ 59 | "-DCMAKE_INSTALL_LIBDIR=" + config.get_miniconda_path() + "/lib", 60 | "-DCMAKE_LINKER=/usr/bin/gold", 61 | "-DCMAKE_BUILD_TYPE=" + build.build_type, 62 | "-DDISABLE_ARCH_NATIVE=ON", 63 | "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", 64 | "-DCMAKE_PREFIX_PATH=" + build.cling_install_path, 65 | '-DCMAKE_CXX_FLAGS="-I ' + build.cling_install_path + '/include"', 66 | ] 67 | 68 | if config.build_libcxx: 69 | cmake_opts = add_libcxx_cmake_arg(cmake_opts) 70 | 71 | cmake_conf = CMakeBuild(prefix=config.get_miniconda_path()) 72 | cm.append( 73 | cmake_conf.configure_step( 74 | build_directory=build.build_path, 75 | directory=config.build_prefix + "/xeus-cling", 76 | opts=cmake_opts, 77 | ) 78 | ) 79 | cm.append( 80 | cmake_conf.build_step( 81 | parallel=config.get_cmake_compiler_threads(), target="install" 82 | ) 83 | ) 84 | 85 | if not config.keep_build: 86 | for build in config.get_xeus_cling_build(): 87 | config.paths_to_delete.append(build.build_path) 88 | config.paths_to_delete.append(config.build_prefix + "/xeus-cling") 89 | 90 | return cm 91 | --------------------------------------------------------------------------------