├── .github ├── ISSUE_TEMPLATE │ └── ci_request.md ├── test.py └── workflows │ ├── ci.yml │ ├── deploy.yml │ ├── sync-project.yaml │ └── tests.yml ├── .gitignore ├── .gitlab ├── README.md ├── anl-ci-pip.yml ├── anl-ci.yml ├── conda-requirements-ppc64le.yml ├── conda-requirements.yml ├── llnl-ci-conda.yml ├── llnl-ci-pip.yml ├── llnl-ci-spack.yml ├── llnl-ci.yml ├── llnl-lassen-openssl-parsl.sh ├── nersc-ci-conda.yml ├── nersc-ci-gitlab-runner.sh ├── nersc-ci-pip.yml ├── nersc-ci-scrontab.example ├── nersc-ci.yml ├── nersc-manual-gitlab-runner-conda.yml ├── nersc-manual-gitlab-runner-pip.yml ├── ornl-ci-conda.yml ├── ornl-ci-pip.yml ├── ornl-ci-spack.yml ├── ornl-ci.yml └── pip-requirements.txt ├── .mergify.yml ├── .readthedocs.yml ├── CONTRIBUTING.md ├── POLICIES.md ├── README.adoc ├── ci ├── config │ └── llnl │ │ ├── compilers.yaml │ │ ├── config.yaml │ │ └── packages.yaml └── tests │ ├── flux │ └── test.sh │ ├── maestro │ ├── study.yaml │ └── test.sh │ ├── parsl-flux │ └── test.sh │ ├── parsl │ ├── test.py │ └── test.sh │ ├── rp-flux │ ├── resource_flux.json │ └── test.sh │ ├── rp-parsl │ └── test.sh │ ├── rp │ └── test.sh │ ├── smartsim │ ├── test.py │ └── test.sh │ ├── swift-t │ └── test.sh │ └── test.py ├── docker ├── base │ ├── centos7 │ │ └── Dockerfile │ ├── rockylinux8 │ │ └── Dockerfile │ ├── scripts │ │ ├── environment.yml │ │ ├── install-mpi.sh │ │ ├── install-python-env.sh │ │ └── install-python.sh │ └── ubuntu2004 │ │ └── Dockerfile ├── flux │ ├── Dockerfile │ ├── checks-annotate.sh │ └── test.sh ├── integration │ ├── Dockerfile │ ├── parsl-flux.sh │ ├── resource_flux.json │ └── rp-flux.sh ├── parsl │ ├── Dockerfile │ ├── test.py │ └── test.sh ├── rct │ ├── Dockerfile │ ├── run_re_test.sh │ └── run_rp_test.sh ├── swift-t │ ├── Dockerfile │ ├── README.adoc │ ├── install-deps.sh │ ├── install-swift-t.sh │ └── test-sanity.sh └── tutorials │ ├── Dockerfile │ ├── README.md │ ├── build.sh │ ├── push-multi-platform.sh │ ├── run-local.sh │ └── run.sh └── docs ├── Makefile ├── make.bat └── source ├── _static └── css │ └── custom.css ├── components.rst ├── conf.py ├── contribute.rst ├── favicon.ico ├── index.rst ├── packaging.rst ├── requirements.txt ├── tutorials.rst └── tutorials ├── docker_image.ipynb ├── images ├── entk-pst-model.png ├── parsl_workflow.svg ├── pi.png ├── psij_job_states.png └── psij_overview.png ├── parsl-rp.ipynb ├── parsl.ipynb ├── psij.ipynb ├── rct.ipynb └── swift-t.ipynb /.github/ISSUE_TEMPLATE/ci_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Request to include a software tool into the CI 3 | about: Use this template to create a request for including a software tool into CI pipelines for one or more facilities. 4 | title: "[CI Request]: [SOFTWARE TOOL NAME]" 5 | labels: "Continuous Integration" 6 | assignees: "mtitov, danlaney" 7 | --- 8 | 9 | Please fullfil the initial requirements described in [Contributing to SDK](https://exaworkssdk.readthedocs.io/en/latest/contribute.html). 10 | 11 | 1. **CI Request** includes the following information about the software tool: 12 | 13 | - [ ] Description of the tool 14 | - [ ] GitHub repository (URL) 15 | - [ ] Supported package managers: **pip**, **conda**, **spack** 16 | - [ ] Specific version to be used (i.e., "stable") and possibility to run 17 | **the latest** version is optional 18 | - [ ] List of all dependencies 19 | - Direct dependencies 20 | - Transitive dependencies (dependencies of dependencies) 21 | - [ ] CI facility: **preferred ones** or **all available** 22 | 23 | 2. After this request is approved, you will be notified and asked to create a **Pull Request** with basic test recipes packed into `test.sh` inside the directory `ci/tests/`. 24 | 25 | 3. Once all the steps above have been completed, your tool will be activated in all the requested CI pipelines. 26 | -------------------------------------------------------------------------------- /.github/test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import requests 3 | from datetime import datetime 4 | import random 5 | import pprint 6 | import subprocess 7 | import sys 8 | import os 9 | 10 | tag = os.getenv("tag").split("_") 11 | run_id = os.getenv("run_id") 12 | branch = os.getenv("branch") 13 | url = 'https://sdk.testing.exaworks.org/result' 14 | 15 | config = { "maintainer_email" : "morton30@llnl.gov", "repo":"sdk"} 16 | 17 | extras = { "config" : config, 18 | "Base Image": tag[0], 19 | "Package Manager": tag[1], 20 | "MPI Flavor": tag[2], 21 | "Python Version": tag[3], 22 | "git_branch" : branch, 23 | "start_time" : str(datetime.now()) 24 | } 25 | 26 | 27 | data = { "run_id" : run_id, 28 | "branch": branch, 29 | } 30 | 31 | def get_conf(): 32 | results = {} 33 | data.update( {"test_name" : "Set Environment", 34 | "results" : results, 35 | "extras": extras, 36 | "test_start_time": str(datetime.now()), 37 | "test_end_time" : str(datetime.now()), 38 | 'function' : '_discover_environment', 39 | "module" : '_conftest' }) 40 | return data 41 | 42 | 43 | def get_result(command, name): 44 | if not name: 45 | name = command.split()[0] 46 | start = str(datetime.now()) 47 | 48 | try: 49 | out = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT).decode("utf-8") 50 | ret = True 51 | except subprocess.CalledProcessError as exc: 52 | out = exc.output.decode("utf-8") 53 | ret = False 54 | 55 | end = str(datetime.now()) 56 | results = { name: 57 | {"passed" : ret}, 58 | } 59 | data.update({ "test_name" : name, 60 | "results" : results, 61 | "test_start_time": start, 62 | "test_end_time" : end, 63 | "extras": {}, 64 | "function" : name, 65 | "module" : "Sanity Checks", 66 | "stdout" : out 67 | }) 68 | return data 69 | 70 | def get_end(): 71 | results = {} 72 | data.update( {"test_name" : "Set Environment", 73 | "results" : results, 74 | "extras": extras, 75 | "test_start_time": str(datetime.now()), 76 | "test_end_time" : str(datetime.now()), 77 | 'function' : '_discover_environment', 78 | "module" : '_end' }) 79 | 80 | return data 81 | 82 | def get_args(): 83 | parser = argparse.ArgumentParser(description='Runs SDK Tests by passing in shell commands') 84 | parser.add_argument('-c', '--command', action='store', type=str, default=None, 85 | help='The command in which you want to test.') 86 | parser.add_argument('-n', '--name', action='store', type=str, default=None, 87 | help='The name of the test.') 88 | parser.add_argument('-s', '--start', action="store_true", default=False, 89 | help='Start a series of test runs with the same id') 90 | parser.add_argument('-e', '--end', action="store_true", default=False, 91 | help='End a series of test runs with the same id') 92 | args = parser.parse_args(sys.argv[1:]) 93 | return args 94 | 95 | def main(): 96 | args = get_args() 97 | 98 | ret = True 99 | if args.start: 100 | data = get_conf() 101 | elif args.end: 102 | data = get_end() 103 | elif args.command: 104 | data = get_result(args.command, args.name) 105 | else: 106 | print("No viable option called, Exiting") 107 | exit(1) 108 | 109 | msg = {"id" : "Github Actions", "key" : "42", "data" : data} 110 | requests.post(url, json=msg) 111 | 112 | if __name__ == '__main__': 113 | main() 114 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI Test 2 | on: 3 | pull_request: 4 | push: { branches: [master] } 5 | 6 | jobs: 7 | test: 8 | name: Run test suite 9 | runs-on: ubuntu-latest 10 | 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | dockerbase: ["centos7", "rockylinux8", "ubuntu2004"] 15 | packagemanager: ["pip"] 16 | mpi: ["openmpi", "mpich"] 17 | pyversion: ["3.8", "3.9"] 18 | env: 19 | DOCKER_TAG: "${{ matrix.dockerbase }}_${{ matrix.packagemanager }}_${{ matrix.mpi }}_${{ matrix.pyversion }}" 20 | 21 | steps: 22 | - name: Checkout repo 23 | uses: actions/checkout@v2 24 | 25 | - name: Build docker images (${{ env.DOCKER_TAG }}) 26 | run: | 27 | docker build \ 28 | -t exaworks/sdk-base:${{ env.DOCKER_TAG }} \ 29 | --cache-from exaworks/sdk-base:${{ env.DOCKER_TAG }} \ 30 | -f docker/base/${{ matrix.dockerbase }}/Dockerfile \ 31 | --build-arg PACKAGE_MANAGER=${{ matrix.packagemanager }} \ 32 | --build-arg PYTHON_VERSION=${{ matrix.pyversion }} \ 33 | --build-arg MPI=${{ matrix.mpi }} \ 34 | docker/base/ 35 | docker build \ 36 | -t rct:${{ env.DOCKER_TAG }} \ 37 | --build-arg BASE_IMAGE=exaworks/sdk-base:${{ env.DOCKER_TAG }} \ 38 | docker/rct 39 | docker build \ 40 | -t rct_parsl:${{ env.DOCKER_TAG }} \ 41 | --build-arg BASE_IMAGE=rct:${{ env.DOCKER_TAG }} \ 42 | docker/parsl 43 | docker build \ 44 | -t rct_parsl_swift:${{ env.DOCKER_TAG }} \ 45 | --build-arg BASE_IMAGE=rct_parsl:${{ env.DOCKER_TAG }} \ 46 | docker/swift-t 47 | docker build \ 48 | -t rct_parsl_swift_flux:${{ env.DOCKER_TAG }} \ 49 | --build-arg BASE_IMAGE=rct_parsl_swift:${{ env.DOCKER_TAG }} \ 50 | docker/flux 51 | docker build \ 52 | -t exaworks_sdk:${{ env.DOCKER_TAG }} \ 53 | --build-arg BASE_IMAGE=rct_parsl_swift_flux:${{ env.DOCKER_TAG }} \ 54 | docker/integration 55 | 56 | - name: Run tests (${{ env.DOCKER_TAG }}) 57 | run: | 58 | total=0 59 | export tag=${{ env.DOCKER_TAG }} 60 | for core in flux parsl rp re swift parsl-flux rp-flux 61 | do 62 | docker run \ 63 | exaworks_sdk:${{ env.DOCKER_TAG }} \ 64 | bash --login -c /tests/$core/test.sh 65 | ret=$? 66 | echo "$core : $ret" 67 | total=$(($total + $ret)) 68 | done 69 | exit $total 70 | 71 | check-pr: 72 | name: validate commits 73 | runs-on: ubuntu-latest 74 | steps: 75 | - uses: actions/checkout@v2 76 | with: 77 | ref: ${{ github.event.pull_request.head.sha }} 78 | fetch-depth: 0 79 | - run: git fetch origin master 80 | - uses: exaworks/pr-validator@master 81 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy 2 | on: 3 | push: { branches: master } 4 | 5 | jobs: 6 | test: 7 | name: Deploy docker images 8 | runs-on: ubuntu-latest 9 | 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | dockerbase: ["centos7", "rockylinux8", "ubuntu2004"] 14 | packagemanager: ["pip"] 15 | mpi: ["openmpi", "mpich"] 16 | pyversion: ["3.7", "3.8", "3.9"] 17 | env: 18 | DOCKER_TAG: "${{ matrix.dockerbase }}_${{ matrix.packagemanager }}_${{ matrix.mpi }}_${{ matrix.pyversion }}" 19 | 20 | steps: 21 | - name: Checkout repo 22 | uses: actions/checkout@v2 23 | 24 | - name: Login to Docker Hub 25 | uses: docker/login-action@v1 26 | with: 27 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 28 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 29 | 30 | - name: Login to GitHub Container Registry 31 | uses: docker/login-action@v1 32 | with: 33 | registry: ghcr.io 34 | username: ${{ github.repository_owner }} 35 | password: ${{ secrets.GITHUB_TOKEN }} 36 | 37 | - name: Set up Docker Buildx 38 | id: buildx 39 | uses: docker/setup-buildx-action@v1 40 | 41 | - name: Build and push base image (${{ env.DOCKER_TAG }}) 42 | id: docker_build 43 | uses: docker/build-push-action@v2 44 | with: 45 | context: ./docker/base/ 46 | file: ./docker/base/${{ matrix.dockerbase }}/Dockerfile 47 | push: true 48 | build-args: | 49 | PACKAGE_MANAGER=${{ matrix.packagemanager }} 50 | MPI=${{ matrix.mpi }} 51 | tags: | 52 | exaworks/sdk-base:${{ env.DOCKER_TAG }} 53 | platforms: linux/amd64 54 | 55 | - name: Build and push composite image (${{ env.DOCKER_TAG }}) 56 | run: | 57 | docker build \ 58 | -t rct:${{ env.DOCKER_TAG }} \ 59 | --build-arg BASE_IMAGE=exaworks/sdk-base:${{ env.DOCKER_TAG }} \ 60 | docker/rct 61 | docker build \ 62 | -t rct_parsl:${{ env.DOCKER_TAG }} \ 63 | --build-arg BASE_IMAGE=rct:${{ env.DOCKER_TAG }} \ 64 | docker/parsl 65 | docker build \ 66 | -t rct_parsl_swift:${{ env.DOCKER_TAG }} \ 67 | --build-arg BASE_IMAGE=rct_parsl:${{ env.DOCKER_TAG }} \ 68 | docker/swift-t 69 | docker build \ 70 | -t rct_parsl_swift_flux:${{ env.DOCKER_TAG }} \ 71 | --build-arg BASE_IMAGE=rct_parsl_swift:${{ env.DOCKER_TAG }} \ 72 | docker/flux 73 | docker build \ 74 | -t exaworks/sdk:${{ env.DOCKER_TAG }} \ 75 | --build-arg BASE_IMAGE=rct_parsl_swift_flux:${{ env.DOCKER_TAG }} \ 76 | docker/integration 77 | docker push exaworks/sdk:${{ env.DOCKER_TAG }} 78 | 79 | - name: Push images with 'latest' tag 80 | if: ${{ matrix.dockerbase == 'centos7' && matrix.packagemanager == 'pip' && matrix.mpi == 'openmpi' && matrix.pyversion == '3.8'}} 81 | run: | 82 | docker tag exaworks/sdk-base:${{ env.DOCKER_TAG }} exaworks/sdk-base:latest 83 | docker push exaworks/sdk-base:latest 84 | docker tag exaworks/sdk:${{ env.DOCKER_TAG }} exaworks/sdk:latest 85 | docker push exaworks/sdk:latest 86 | -------------------------------------------------------------------------------- /.github/workflows/sync-project.yaml: -------------------------------------------------------------------------------- 1 | name: Project automations 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | pull_request: 8 | types: 9 | - opened 10 | 11 | jobs: 12 | issue_opened: 13 | name: issue_opened 14 | runs-on: ubuntu-latest 15 | if: github.event_name == 'issues' && github.event.action == 'opened' 16 | steps: 17 | - name: 'Move issue to Todo' 18 | uses: leonsteinhaeuser/project-beta-automations@v1.2.0 19 | with: 20 | gh_app_secret_key: ${{ secrets.GH_APP_SECRET_KEY }} 21 | gh_app_ID: ${{ secrets.GH_APP_ID }} 22 | gh_app_installation_ID: ${{ secrets.GH_APP_INSTALLATION_ID }} 23 | organization: exaworks 24 | project_id: 1 25 | resource_node_id: ${{ github.event.issue.node_id }} 26 | status_value: "Todo" 27 | 28 | pr_opened: 29 | name: pr_opened 30 | runs-on: ubuntu-latest 31 | if: github.event_name == 'pull_request' && github.event.action == 'opened' 32 | steps: 33 | - name: 'Move PR to "In Progress"' 34 | uses: leonsteinhaeuser/project-beta-automations@v1.2.0 35 | with: 36 | gh_app_secret_key: ${{ secrets.GH_APP_SECRET_KEY }} 37 | gh_app_ID: ${{ secrets.GH_APP_ID }} 38 | gh_app_installation_ID: ${{ secrets.GH_APP_INSTALLATION_ID }} 39 | organization: exaworks 40 | project_id: 1 41 | resource_node_id: ${{ github.event.pull_request.node_id }} 42 | status_value: "In Progress" 43 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Scheduled CI Test 2 | on: 3 | schedule: 4 | - cron: '0 20 * * *' 5 | 6 | jobs: 7 | test: 8 | name: Run test suite 9 | runs-on: ubuntu-latest 10 | 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | dockerbase: ["centos7", "rockylinux8", "ubuntu2004"] 15 | packagemanager: ["pip"] 16 | mpi: ["openmpi", "mpich"] 17 | pyversion: ["3.7", "3.8", "3.9"] 18 | env: 19 | DOCKER_TAG: "${{ matrix.dockerbase }}_${{ matrix.packagemanager }}_${{ matrix.mpi }}_${{ matrix.pyversion }}" 20 | 21 | steps: 22 | - name: Checkout repo 23 | uses: actions/checkout@v2 24 | 25 | 26 | - name: Run tests (${{ env.DOCKER_TAG }}) 27 | run: | 28 | export tag=${{ env.DOCKER_TAG }} 29 | export run_id=$RANDOM 30 | export branch=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') 31 | docker pull exaworks/sdk:${{ env.DOCKER_TAG }} 32 | python3 .github/test.py -s 33 | for core in flux parsl rp swift parsl-flux rp-flux 34 | do 35 | python3 .github/test.py -n $core -c \ 36 | "docker run exaworks/sdk:${{ env.DOCKER_TAG }} \ 37 | bash --login -c /tests/$core/test.sh" 38 | done 39 | python3 .github/test.py -e 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | docs/_build/ 3 | .ipynb_checkpoints/ 4 | -------------------------------------------------------------------------------- /.gitlab/README.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | ## Environment variables 4 | 5 | The following environment variables should be set within CI/CD settings: 6 | - `SDK_DASHBOARD_URL` - used to publish results from the executed tests; 7 | - `SDK_DASHBOARD_TOKEN` - unique token per facility (randomly generated UUID); 8 | - `SDK_WORK_DIR` - working directory in a shared space; 9 | - In case of OLCF: `SDK_WORK_DIR_OPEN`, `SDK_WORK_DIR_ALPINE`, 10 | `SDK_WORK_DIR_ORION`; 11 | 12 | ## GitLab working directory 13 | 14 | ```shell 15 | rm -rf ${HOME}/.jacamar-ci 16 | mkdir /gitlab-runner 17 | ln -s /gitlab-runner ${HOME}/.jacamar-ci 18 | chmod 700 /gitlab-runner 19 | ``` 20 | 21 | `` per facility 22 | * LLNL: `/usr/workspace/${USER}` 23 | * OLCF: `$MEMBERWORK/csc449` 24 | * ALCF: TBD... 25 | * NERSC: perlmutter: `/pscratch/sd/[FirstLetterOf${USER}]/${USER}` 26 | 27 | ## SPACK-related issues 28 | 29 | ```shell 30 | # activate spack 31 | . $SPACK_WORK_DIR/spack/share/spack/setup-env.sh 32 | # if issue is related to a certain environment, then activate it 33 | # spack env activate $SPACK_ENV_NAME 34 | ``` 35 | 36 | ### Cached obsolete packages 37 | 38 | Example: `Error: Package 'armpl' not found.` 39 | [#31453](https://github.com/spack/spack/issues/31453) 40 | 41 | ```shell 42 | spack clean -ab 43 | # or `spack clean -fmps` to keep cached downloads 44 | ``` 45 | 46 | OR expanded set of commands 47 | 48 | ```shell 49 | spack clean -a 50 | rm -rf $HOME/.spack/cache 51 | spack providers 52 | ``` 53 | 54 | ### URL fetch method 55 | 56 | Default fetch method is `urllib`, but if there is an issue with downloading 57 | packages, then switch it to `curl`. 58 | 59 | ```shell 60 | spack config add config:url_fetch_method:curl 61 | spack config add config:connect_timeout:30 62 | ``` 63 | 64 | ### Remote package is not accessible 65 | 66 | Use pre-downloaded packages. Below is an example for the `python` package. 67 | ```shell 68 | mkdir -p pre-stage 69 | # download Python tarball locally 70 | # copy (`scp`) Python tarball into `pre-stage` directory 71 | # refer `pre-stage` directory to `spack` 72 | spack stage -p pre-stage python 73 | ``` 74 | 75 | ### LLNL Machines not installing Swift-T 76 | 77 | When spack is installing Swift-T, it installs `ant` as a dependency. 78 | For an unknown reason, `ant` is unable to initialize the Java Virtual Machine, 79 | thus we're unable to install Swift-T. For now a [temporary solution](https://github.com/ExaWorks/SDK/blob/9c272ba2462298ff49e91e97f73fca029bd9c11b/.gitlab/llnl-ci-spack.yml#L55-L59) has been put in place. 80 | We chose to instruct Spack to use the available system install of `ant` instead of a Spack installation. 81 | 82 | Here is the related issue: [#174](https://github.com/ExaWorks/SDK/issues/174) 83 | 84 | The temporary solution can be improved upon once [Spack PR #40976](https://github.com/spack/spack/pull/40976) is merged into Spack 85 | 86 | --- 87 | 88 | ## References for GitLab-related services 89 | 90 | ### LLNL 91 | 92 | * [LC Resources and Environment](https://hpc.llnl.gov/documentation/tutorials/livermore-computing-resources-and-environment) 93 | * [LC GitLab CI](https://lc.llnl.gov/confluence/display/GITLAB/GitLab+CI) 94 | 95 | ### OLCF/ORNL 96 | 97 | * [CI/CD Workflows](https://docs.olcf.ornl.gov/services_and_applications/slate/workflows/overview.html) 98 | * [GitLab Runners](https://docs.olcf.ornl.gov/services_and_applications/slate/use_cases/gitlab_runner.html) 99 | 100 | ### NERSC 101 | 102 | * [GitLab](https://docs.nersc.gov/services/gitlab/) 103 | * [NERSC CI docs](https://software.nersc.gov/NERSC/nersc-ci-docs) 104 | * [Introduction to CI at NERSC](https://www.nersc.gov/assets/Uploads/Introduction-to-CI-at-NERSC-07072021.pdf) 105 | * [GitLab CI Tutorial](https://www.nersc.gov/assets/Uploads/2017-02-06-Gitlab-CI.pdf) 106 | * [Advanced GitLab Tutorial](https://www.nersc.gov/assets/Uploads/Advanced-Gitlab.pdf) 107 | * [Spack GitLab Pipeline](https://docs.nersc.gov/applications/e4s/spack_gitlab_pipeline/) 108 | * [CI examples, tutorials, and general resources](https://software.nersc.gov/ci-resources) 109 | 110 | --- 111 | 112 | -------------------------------------------------------------------------------- /.gitlab/anl-ci-pip.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | PIP_WORK_DIR: ${SDK_WORK_DIR}/pip 3 | PIP_REQUIREMENTS: .gitlab/pip-requirements.txt 4 | 5 | 6 | 7 | stages: 8 | - setup 9 | - build 10 | - test 11 | - cleanup 12 | 13 | 14 | ### MACHINES 15 | 16 | .on_polaris: 17 | variables: 18 | SITE: "polaris" 19 | VENV_ENV_NAME: "polaris-env" 20 | 21 | ### SCRIPTS 22 | .final_steps: 23 | script: &finalize 24 | - chgrp -fR CSC249ADTR01 ${PIP_WORK_DIR} || true 25 | - chmod -fR 02770 ${PIP_WORK_DIR} || true 26 | 27 | .job_tags: 28 | tags: 29 | - $SITE 30 | - $RUNNER_TYPE 31 | 32 | .pip_env_setup: 33 | stage: setup 34 | script: 35 | - mkdir -p ${PIP_WORK_DIR} 36 | - test -d ${PIP_WORK_DIR}/${VENV_ENV_NAME} && exit 0 37 | - python3 -m venv ${PIP_WORK_DIR}/${VENV_ENV_NAME} 38 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 39 | - pip install -U pip setuptools wheel 40 | - pip cache purge 41 | - *finalize 42 | 43 | .pip_build: 44 | stage: build 45 | script: 46 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 47 | - pip install --no-cache-dir -r ${PIP_REQUIREMENTS} 48 | - *finalize 49 | 50 | .pip_test: 51 | stage: test 52 | variables: 53 | MAINTAINER: "wilke@anl.gov" 54 | IM_NUMBER: "" 55 | TESTS_GROUP: "pip" 56 | SITE_ID: "anl-$SITE" 57 | script: 58 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 59 | - python3 ci/tests/test.py -s 60 | - for TEST in flux parsl parsl-flux maestro; do 61 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 62 | done 63 | - python3 ci/tests/test.py -e 64 | - deactivate 65 | 66 | ### JOBS 67 | 68 | # POLARIS 69 | pip_env_setup_polaris: 70 | variables: 71 | RUNNER_TYPE: "shell" 72 | extends: [ .on_polaris, .job_tags, .pip_env_setup ] 73 | 74 | pip_build_polaris: 75 | variables: 76 | RUNNER_TYPE: "shell" 77 | needs: [ pip_env_setup_polaris ] 78 | extends: [ .on_polaris, .job_tags, .pip_build ] 79 | 80 | pip_test_polaris: 81 | variables: 82 | RUNNER_TYPE: "batch" 83 | ANL_POLARIS_SCHEDULER_PARAMETERS: "-A CSC249ADTR01 -q debug -l select=1 -l walltime=45:00 -l filesystems=home:grand" 84 | needs: [ pip_build_polaris ] 85 | extends: [ .on_polaris, .job_tags, .pip_test ] 86 | 87 | 88 | 89 | # ALWAYS 90 | pip_cleanup: 91 | stage: cleanup 92 | when: always 93 | variables: 94 | RUNNER_TYPE: "shell" 95 | extends: [ .on_polaris, .job_tags ] 96 | script: 97 | - *finalize 98 | -------------------------------------------------------------------------------- /.gitlab/anl-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - local: '.gitlab/anl-ci-*.yml' 3 | 4 | stages: 5 | - setup 6 | - build 7 | - test 8 | - cleanup 9 | -------------------------------------------------------------------------------- /.gitlab/conda-requirements-ppc64le.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - conda-forge 3 | dependencies: 4 | - python>=3.7 5 | - parsl 6 | - radical.pilot 7 | - pip 8 | - pip: 9 | - pytest==7.2.2 10 | - pytest-tap==3.3 11 | - requests==2.28.1 12 | -------------------------------------------------------------------------------- /.gitlab/conda-requirements.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - conda-forge 3 | - swift-t 4 | dependencies: 5 | - python>=3.7 6 | - swift-t 7 | - parsl 8 | - radical.pilot 9 | - flux-core 10 | - flux-sched 11 | - pip 12 | - pip: 13 | - pytest==7.2.2 14 | - pytest-tap==3.3 15 | - requests==2.28.1 16 | -------------------------------------------------------------------------------- /.gitlab/llnl-ci-conda.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | CONDA_WORK_DIR: ${SDK_WORK_DIR}/conda 3 | 4 | 5 | stages: 6 | - setup 7 | - build 8 | - test 9 | - cleanup 10 | 11 | 12 | ### MACHINES 13 | .on_quartz: 14 | variables: 15 | SITE: "quartz" 16 | CONDA_ENV_NAME: "quartz-env" 17 | CONDA_ARCH: "x86_64" 18 | CONDA_REQUIREMENTS: .gitlab/conda-requirements.yml 19 | 20 | .on_ruby: 21 | variables: 22 | SITE: "ruby" 23 | CONDA_ENV_NAME: "ruby-env" 24 | CONDA_ARCH: "x86_64" 25 | CONDA_REQUIREMENTS: .gitlab/conda-requirements.yml 26 | 27 | .on_lassen: 28 | variables: 29 | SITE: "lassen" 30 | CONDA_ENV_NAME: "lassen-env" 31 | CONDA_ARCH: "ppc64le" 32 | CONDA_REQUIREMENTS: .gitlab/conda-requirements-ppc64le.yml 33 | 34 | ### SCRIPTS 35 | .final_steps: 36 | script: &finalize 37 | - chgrp -fR exaworks ${CONDA_WORK_DIR} || true 38 | - chmod -fR 02770 ${CONDA_WORK_DIR} || true 39 | 40 | .job_tags: 41 | tags: 42 | - $SITE 43 | - $RUNNER_TYPE 44 | 45 | .conda_setup: 46 | stage: setup 47 | script: 48 | - mkdir -p ${CONDA_WORK_DIR} 49 | - test -d ${CONDA_WORK_DIR}/miniconda3-${SITE} && exit 0 50 | - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 51 | - chmod +x ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 52 | - ${CONDA_WORK_DIR}/miniconda-${SITE}.sh -b -p ${CONDA_WORK_DIR}/miniconda3-${SITE} 53 | - *finalize 54 | 55 | .conda_env_setup: 56 | stage: setup 57 | script: 58 | - test -d ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} && exit 0 59 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 60 | - eval "$(conda shell.posix hook)" 61 | - conda create --prefix ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} -y 62 | - *finalize 63 | 64 | .conda_build: 65 | stage: build 66 | script: 67 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 68 | - eval "$(conda shell.posix hook)" 69 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 70 | - conda env update --file ${CONDA_REQUIREMENTS} 71 | - *finalize 72 | 73 | .conda_test: 74 | stage: test 75 | variables: 76 | MAINTAINER: "arambula2@llnl.gov" 77 | IM_NUMBER: "LLNL-MI-834241" 78 | TESTS_GROUP: "conda" 79 | SITE_ID: "llnl-$SITE" 80 | script: 81 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 82 | - eval "$(conda shell.posix hook)" 83 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 84 | - python3 ci/tests/test.py -s 85 | - for TEST in flux parsl rp swift-t parsl-flux rp-flux; do 86 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 87 | done 88 | - python3 ci/tests/test.py -e 89 | - *finalize 90 | 91 | ### JOBS 92 | 93 | # QUARTZ 94 | conda_setup_quartz: 95 | variables: 96 | RUNNER_TYPE: "shell" 97 | extends: [ .on_quartz, .job_tags, .conda_setup ] 98 | 99 | conda_env_setup_quartz: 100 | variables: 101 | RUNNER_TYPE: "shell" 102 | needs: [ conda_setup_quartz ] 103 | extends: [ .on_quartz, .job_tags, .conda_env_setup ] 104 | 105 | conda_build_quartz: 106 | variables: 107 | RUNNER_TYPE: "shell" 108 | needs: [ conda_env_setup_quartz ] 109 | extends: [ .on_quartz, .job_tags, .conda_build ] 110 | 111 | conda_test_quartz: 112 | variables: 113 | RUNNER_TYPE: "batch" 114 | LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 115 | needs: [ conda_build_quartz ] 116 | extends: [ .on_quartz, .job_tags, .conda_test ] 117 | 118 | 119 | # RUBY 120 | conda_setup_ruby: 121 | variables: 122 | RUNNER_TYPE: "shell" 123 | extends: [ .on_ruby, .job_tags, .conda_setup ] 124 | 125 | conda_env_setup_ruby: 126 | variables: 127 | RUNNER_TYPE: "shell" 128 | needs: [ conda_setup_ruby ] 129 | extends: [ .on_ruby, .job_tags, .conda_env_setup ] 130 | 131 | conda_build_ruby: 132 | variables: 133 | RUNNER_TYPE: "shell" 134 | needs: [ conda_env_setup_ruby ] 135 | extends: [ .on_ruby, .job_tags, .conda_build ] 136 | 137 | conda_test_ruby: 138 | variables: 139 | RUNNER_TYPE: "batch" 140 | LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 141 | needs: [ conda_build_ruby ] 142 | extends: [ .on_ruby, .job_tags, .conda_test ] 143 | 144 | 145 | # LASSEN 146 | conda_setup_lassen: 147 | variables: 148 | RUNNER_TYPE: "shell" 149 | extends: [ .on_lassen, .job_tags, .conda_setup ] 150 | 151 | conda_env_setup_lassen: 152 | variables: 153 | RUNNER_TYPE: "shell" 154 | needs: [ conda_setup_lassen ] 155 | extends: [ .on_lassen, .job_tags, .conda_env_setup ] 156 | 157 | conda_build_lassen: 158 | variables: 159 | RUNNER_TYPE: "shell" 160 | needs: [ conda_env_setup_lassen ] 161 | extends: [ .on_lassen, .job_tags, .conda_build ] 162 | 163 | conda_test_lassen: 164 | variables: 165 | RUNNER_TYPE: "batch" 166 | LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 45" 167 | LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD" 168 | ENVIRONMENT: "BATCH" 169 | needs: [ conda_build_lassen ] 170 | extends: [ .on_lassen, .job_tags, .conda_test ] 171 | 172 | 173 | # ALWAYS 174 | conda_cleanup: 175 | stage: cleanup 176 | when: always 177 | variables: 178 | RUNNER_TYPE: "shell" 179 | extends: [ .on_quartz, .job_tags ] 180 | script: 181 | - *finalize 182 | 183 | -------------------------------------------------------------------------------- /.gitlab/llnl-ci-pip.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | PIP_WORK_DIR: ${SDK_WORK_DIR}/pip 3 | PIP_REQUIREMENTS: .gitlab/pip-requirements.txt 4 | 5 | 6 | stages: 7 | - setup 8 | - build 9 | - test 10 | - cleanup 11 | 12 | 13 | ### MACHINES 14 | .on_quartz: 15 | variables: 16 | SITE: "quartz" 17 | VENV_ENV_NAME: "quartz-env" 18 | PYTHON_MODULE: "python/3.9.12" 19 | 20 | .on_ruby: 21 | variables: 22 | SITE: "ruby" 23 | VENV_ENV_NAME: "ruby-env" 24 | PYTHON_MODULE: "python/3.9.12" 25 | 26 | .on_lassen: 27 | variables: 28 | SITE: "lassen" 29 | VENV_ENV_NAME: "lassen-env" 30 | PYTHON_MODULE: "python/3.8.2" 31 | 32 | 33 | ### SCRIPTS 34 | .final_steps: 35 | script: &finalize 36 | - chgrp -fR exaworks ${PIP_WORK_DIR} || true 37 | - chmod -fR 02770 ${PIP_WORK_DIR} || true 38 | 39 | .job_tags: 40 | tags: 41 | - $SITE 42 | - $RUNNER_TYPE 43 | 44 | .pip_env_setup: 45 | stage: setup 46 | script: 47 | - mkdir -p ${PIP_WORK_DIR} 48 | - test -d ${PIP_WORK_DIR}/${VENV_ENV_NAME} && exit 0 49 | - module add ${PYTHON_MODULE} 50 | - python3 -m venv ${PIP_WORK_DIR}/${VENV_ENV_NAME} 51 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 52 | - pip install -U pip setuptools wheel 53 | - pip cache purge 54 | - *finalize 55 | 56 | .pip_build: 57 | stage: build 58 | script: 59 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 60 | - pip install --no-cache-dir -r ${PIP_REQUIREMENTS} 61 | - (smart clean && smart build) > /dev/null 2>&1 || true 62 | - *finalize 63 | 64 | .pip_test: 65 | stage: test 66 | variables: 67 | MAINTAINER: "arambula2@llnl.gov" 68 | IM_NUMBER: "LLNL-MI-834241" 69 | TESTS_GROUP: "pip" 70 | SITE_ID: "llnl-$SITE" 71 | script: 72 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 73 | - python3 ci/tests/test.py -s 74 | - for TEST in flux parsl rp parsl-flux rp-flux maestro smartsim; do 75 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 76 | done 77 | - python3 ci/tests/test.py -e 78 | - deactivate 79 | 80 | ### JOBS 81 | 82 | # QUARTZ 83 | pip_env_setup_quartz: 84 | variables: 85 | RUNNER_TYPE: "shell" 86 | extends: [ .on_quartz, .job_tags, .pip_env_setup ] 87 | 88 | pip_build_quartz: 89 | variables: 90 | RUNNER_TYPE: "shell" 91 | needs: [ pip_env_setup_quartz ] 92 | extends: [ .on_quartz, .job_tags, .pip_build ] 93 | 94 | pip_test_quartz: 95 | variables: 96 | RUNNER_TYPE: "batch" 97 | LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 98 | needs: [ pip_build_quartz ] 99 | extends: [ .on_quartz, .job_tags, .pip_test ] 100 | 101 | 102 | # RUBY 103 | pip_env_setup_ruby: 104 | variables: 105 | RUNNER_TYPE: "shell" 106 | extends: [ .on_ruby, .job_tags, .pip_env_setup ] 107 | 108 | pip_build_ruby: 109 | variables: 110 | RUNNER_TYPE: "shell" 111 | needs: [ pip_env_setup_ruby ] 112 | extends: [ .on_ruby, .job_tags, .pip_build ] 113 | 114 | pip_test_ruby: 115 | variables: 116 | RUNNER_TYPE: "batch" 117 | LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 118 | needs: [ pip_build_ruby ] 119 | extends: [ .on_ruby, .job_tags, .pip_test ] 120 | 121 | 122 | # LASSEN 123 | pip_env_setup_lassen: 124 | variables: 125 | RUNNER_TYPE: "shell" 126 | extends: [ .on_lassen, .job_tags, .pip_env_setup ] 127 | 128 | pip_build_lassen: 129 | variables: 130 | RUNNER_TYPE: "shell" 131 | needs: [ pip_env_setup_lassen ] 132 | extends: [ .on_lassen, .job_tags, .pip_build ] 133 | before_script: 134 | - .gitlab/llnl-lassen-openssl-parsl.sh ${PIP_WORK_DIR}/${VENV_ENV_NAME} 135 | 136 | pip_test_lassen: 137 | variables: 138 | RUNNER_TYPE: "batch" 139 | LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 45" 140 | LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD" 141 | ENVIRONMENT: "BATCH" 142 | needs: [ pip_build_lassen ] 143 | extends: [ .on_lassen, .job_tags, .pip_test ] 144 | 145 | 146 | # ALWAYS 147 | pip_cleanup: 148 | stage: cleanup 149 | when: always 150 | variables: 151 | RUNNER_TYPE: "shell" 152 | extends: [ .on_quartz, .job_tags ] 153 | script: 154 | - *finalize 155 | 156 | -------------------------------------------------------------------------------- /.gitlab/llnl-ci-spack.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | SPACK_WORK_DIR: ${SDK_WORK_DIR}/spack 3 | 4 | 5 | stages: 6 | - setup 7 | - build 8 | - test 9 | - cleanup 10 | 11 | 12 | ### MACHINES 13 | .on_quartz: 14 | variables: 15 | SITE: "quartz" 16 | SPACK_ENV_NAME: "rhel7-broadwell" 17 | TARGET_ARCH_OPT: "target=x86_64" 18 | 19 | .on_ruby: 20 | variables: 21 | SITE: "ruby" 22 | SPACK_ENV_NAME: "rhel7-cascadelake" 23 | TARGET_ARCH_OPT: "" 24 | 25 | .on_lassen: 26 | variables: 27 | SITE: "lassen" 28 | SPACK_ENV_NAME: "rhel7-ppc64le" 29 | TARGET_ARCH_OPT: "target=ppc64le" 30 | 31 | 32 | ### SCRIPTS 33 | .final_steps: 34 | script: &finalize 35 | - chgrp -fR exaworks ${SPACK_WORK_DIR} || true 36 | - chmod -fR 02770 ${SPACK_WORK_DIR} || true 37 | 38 | .job_tags: 39 | tags: 40 | - $SITE 41 | - $RUNNER_TYPE 42 | 43 | .spack_env_setup: 44 | stage: setup 45 | script: 46 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 47 | - test "$(spack env list | grep $SPACK_ENV_NAME)" && exit 0 48 | - spack env create ${SPACK_ENV_NAME} 49 | - for i in {1..3}; do 50 | spack env activate ${SPACK_ENV_NAME} && break || sleep 5; 51 | done 52 | - spack config add concretizer:unify:when_possible 53 | - spack config add concretizer:reuse:false 54 | - spack config add config:db_lock_timeout:300 55 | - spack config add packages:ant:buildable:false 56 | - spack config add packages:ant:externals:spec:ant@1.10.5 57 | # - spack config add packages:ant:externals:prefix:"/usr" # FIXME/NOTE: uncomment once spack PR: https://github.com/spack/spack/pull/40976 is merged 58 | - > 59 | sed -i '/- spec: ant@1\.10\.5/ a \ \ \ \ \ \ \ prefix: /usr' 60 | $(find $(spack location -e ${SPACK_ENV_NAME}) -name spack.yaml -print -quit) # NOTE: This is a temporary fix until pr above is merged 61 | - module --latest load gcc && spack compiler find 62 | - spack add rust ^openssl certs=system 63 | - spack install || (spack env deactivate && spack env remove ${SPACK_ENV_NAME} -y && exit 1) 64 | - *finalize 65 | 66 | .spack_build: 67 | stage: build 68 | script: 69 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 70 | - for i in {1..3}; do 71 | spack env activate ${SPACK_ENV_NAME} && break || sleep 5; 72 | done 73 | - spack uninstall -y exaworks py-pytest || true 74 | - spack add exaworks ^python@3.9 py-pytest 75 | - spack concretize -f 76 | - spack install 77 | - *finalize 78 | 79 | .spack_test: 80 | stage: test 81 | variables: 82 | MAINTAINER: "arambula2@llnl.gov" 83 | IM_NUMBER: "LLNL-MI-834241" 84 | TESTS_GROUP: "spack" 85 | SITE_ID: "llnl-$SITE" 86 | script: 87 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 88 | - for i in {1..3}; do 89 | spack env activate ${SPACK_ENV_NAME} && break || sleep 5; 90 | done 91 | - spack load exaworks py-pytest 92 | - python3 ci/tests/test.py -s 93 | - for TEST in flux parsl rp swift-t parsl-flux rp-flux; do 94 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 95 | done 96 | - python3 ci/tests/test.py -e 97 | 98 | ### JOBS 99 | 100 | spack_setup: 101 | stage: setup 102 | variables: 103 | RUNNER_TYPE: "shell" 104 | extends: [ .on_ruby, .job_tags ] 105 | script: 106 | - test -d ${SPACK_WORK_DIR} && exit 0 107 | - git clone -c feature.manyFiles=true -c core.sharedRepository=true https://github.com/spack/spack.git ${SPACK_WORK_DIR} 108 | - *finalize 109 | 110 | spack_update: 111 | stage: build 112 | variables: 113 | RUNNER_TYPE: "shell" 114 | extends: [ .on_ruby, .job_tags ] 115 | needs: [ spack_env_setup_ruby, spack_env_setup_quartz, spack_env_setup_lassen ] 116 | script: 117 | - cd ${SPACK_WORK_DIR} 118 | - git checkout HEAD^ . 119 | - git reset --hard HEAD 120 | - git pull --ff 121 | - *finalize 122 | 123 | 124 | # QUARTZ 125 | spack_env_setup_quartz: 126 | variables: 127 | RUNNER_TYPE: "shell" 128 | needs: [ spack_setup ] 129 | extends: [ .on_quartz, .job_tags, .spack_env_setup ] 130 | 131 | spack_build_quartz: 132 | variables: 133 | RUNNER_TYPE: "shell" 134 | needs: [ spack_update ] 135 | extends: [ .on_quartz, .job_tags, .spack_build ] 136 | 137 | spack_test_quartz: 138 | variables: 139 | RUNNER_TYPE: "batch" 140 | LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 141 | needs: [ spack_build_quartz ] 142 | extends: [ .on_quartz, .job_tags, .spack_test ] 143 | 144 | 145 | # RUBY 146 | spack_env_setup_ruby: 147 | variables: 148 | RUNNER_TYPE: "shell" 149 | needs: [ spack_setup ] 150 | extends: [ .on_ruby, .job_tags, .spack_env_setup ] 151 | 152 | spack_build_ruby: 153 | variables: 154 | RUNNER_TYPE: "shell" 155 | needs: [ spack_update ] 156 | extends: [ .on_ruby, .job_tags, .spack_build ] 157 | 158 | spack_test_ruby: 159 | variables: 160 | RUNNER_TYPE: "batch" 161 | LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 162 | needs: [ spack_build_ruby ] 163 | extends: [ .on_ruby, .job_tags, .spack_test ] 164 | 165 | 166 | # LASSEN 167 | spack_env_setup_lassen: 168 | variables: 169 | RUNNER_TYPE: "shell" 170 | needs: [ spack_setup ] 171 | extends: [ .on_lassen, .job_tags, .spack_env_setup ] 172 | 173 | spack_build_lassen: 174 | variables: 175 | RUNNER_TYPE: "shell" 176 | needs: [ spack_update ] 177 | extends: [ .on_lassen, .job_tags, .spack_build ] 178 | 179 | spack_test_lassen: 180 | variables: 181 | RUNNER_TYPE: "batch" 182 | LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 45" 183 | LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD" 184 | ENVIRONMENT: "BATCH" 185 | needs: [ spack_build_lassen ] 186 | extends: [ .on_lassen, .job_tags, .spack_test ] 187 | 188 | 189 | # ALWAYS 190 | spack_cleanup: 191 | stage: cleanup 192 | when: always 193 | variables: 194 | RUNNER_TYPE: "shell" 195 | extends: [ .on_ruby, .job_tags ] 196 | script: 197 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 198 | - spack clean -a || true 199 | - rm -rf ${HOME}/.spack/cache 200 | - spack providers 201 | - find ${SPACK_WORK_DIR} -maxdepth 0 -type d -ctime +5 | xargs rm -rf 202 | - *finalize 203 | 204 | -------------------------------------------------------------------------------- /.gitlab/llnl-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - local: '.gitlab/llnl-ci-*.yml' 3 | 4 | stages: 5 | - setup 6 | - build 7 | - test 8 | - cleanup 9 | -------------------------------------------------------------------------------- /.gitlab/llnl-lassen-openssl-parsl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | VENV_NAME="$1" 6 | 7 | OPENSSL_VERSION="1.1.1v" 8 | OPENSSL_PREFIX="${VENV_NAME}/../lassen-openssl" 9 | 10 | if [[ -d "$OPENSSL_PREFIX" ]]; then 11 | echo 'OpenSSL for Lassen is already installed' 12 | exit 13 | fi 14 | 15 | source "${VENV_NAME}/bin/activate" 16 | pip install -U pip setuptools wheel 17 | 18 | curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz 19 | tar xvf openssl-${OPENSSL_VERSION}.tar.gz 20 | cd openssl-${OPENSSL_VERSION} 21 | ./config no-shared no-ssl2 no-ssl3 -fPIC --prefix="${OPENSSL_PREFIX}" 22 | make && make install 23 | cd .. 24 | 25 | CFLAGS="-I${OPENSSL_PREFIX}/include" LDFLAGS="-L${OPENSSL_PREFIX}/lib" \ 26 | pip install --no-cache-dir --no-binary :all: cryptography==3.3.2 27 | 28 | pip cache purge 29 | rm -rf openssl-${OPENSSL_VERSION}* 30 | 31 | -------------------------------------------------------------------------------- /.gitlab/nersc-ci-conda.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | CONDA_WORK_DIR: ${SDK_WORK_DIR}/conda 3 | CONDA_REQUIREMENTS: .gitlab/conda-requirements.yml 4 | 5 | 6 | stages: 7 | - setup 8 | - build 9 | - test 10 | - cleanup 11 | 12 | 13 | ### MACHINES 14 | .on_perlmutter: 15 | variables: 16 | SITE: "perlmutter" 17 | CONDA_ENV_NAME: "perlmutter-env" 18 | CONDA_ARCH: "x86_64" 19 | 20 | 21 | ### SCRIPTS 22 | .final_steps: 23 | script: &finalize 24 | - chgrp -fR m3973 ${CONDA_WORK_DIR} || true 25 | - chmod -fR 02770 ${CONDA_WORK_DIR} || true 26 | 27 | .job_tags: 28 | tags: 29 | - $SITE 30 | - $RUNNER_TYPE 31 | 32 | .conda_setup: 33 | stage: setup 34 | script: 35 | - mkdir -p ${CONDA_WORK_DIR} 36 | - test -d ${CONDA_WORK_DIR}/miniconda3-${SITE} && exit 0 37 | - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 38 | - chmod +x ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 39 | ! - ${CONDA_WORK_DIR}/miniconda-${SITE}.sh -b -p ${CONDA_WORK_DIR}/miniconda3-${SITE} 40 | - *finalize 41 | 42 | .conda_env_setup: 43 | stage: setup 44 | script: 45 | - test -d ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} && exit 0 46 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 47 | - eval "$(conda shell.posix hook)" 48 | - conda create --prefix ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} -y 49 | - *finalize 50 | 51 | .conda_build: 52 | stage: build 53 | script: 54 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 55 | - eval "$(conda shell.posix hook)" 56 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 57 | - conda env update --file ${CONDA_REQUIREMENTS} 58 | - *finalize 59 | 60 | .conda_test: 61 | stage: test 62 | variables: 63 | MAINTAINER: "cowan@bnl.gov" 64 | IM_NUMBER: "" 65 | TESTS_GROUP: "conda" 66 | SITE_ID: "nersc-$SITE" 67 | script: 68 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 69 | - eval "$(conda shell.posix hook)" 70 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 71 | - python3 ci/tests/test.py -s 72 | - for TEST in parsl rp swift-t; do 73 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 74 | done 75 | - python3 ci/tests/test.py -e 76 | - *finalize 77 | 78 | ### JOBS 79 | 80 | # PERLMUTTER 81 | conda_setup_perlmutter: 82 | variables: 83 | RUNNER_TYPE: "shell" 84 | extends: [ .on_perlmutter, .job_tags, .conda_setup ] 85 | 86 | conda_env_setup_perlmutter: 87 | variables: 88 | RUNNER_TYPE: "shell" 89 | needs: [ conda_setup_perlmutter ] 90 | extends: [ .on_perlmutter, .job_tags, .conda_env_setup ] 91 | 92 | conda_build_perlmutter: 93 | variables: 94 | RUNNER_TYPE: "shell" 95 | needs: [ conda_env_setup_perlmutter ] 96 | extends: [ .on_perlmutter, .job_tags, .conda_build ] 97 | 98 | conda_test_perlmutter: 99 | variables: 100 | RUNNER_TYPE: "batch" 101 | SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 102 | needs: [ conda_build_perlmutter ] 103 | extends: [ .on_perlmutter, .job_tags, .conda_test ] 104 | 105 | 106 | # ALWAYS 107 | conda_cleanup: 108 | stage: cleanup 109 | when: always 110 | variables: 111 | RUNNER_TYPE: "shell" 112 | extends: [ .on_perlmutter, .job_tags ] 113 | script: 114 | - *finalize 115 | 116 | -------------------------------------------------------------------------------- /.gitlab/nersc-ci-gitlab-runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Used for manually running tests via 'gitlab-runner exec shell ...' 4 | # (eg. via scrontab) when no runner is available for direct gitlab 5 | # integration. 6 | 7 | # prereqs: 8 | # - install gitlab-runner. Don't need root, can do like: 9 | # mkdir -p ~/packages/gitlab-runner 10 | # cd ~/packages 11 | # curl -LJO "https://gitlab-runner-downloads.s3.amazonaws.com/latest/rpm/gitlab-runner_amd64.rpm" 12 | # cd gitlab-runner 13 | # rpm2cpio ../gitlab-runner_amd64.rpm | cpio -i --make-directories 14 | # and then run gitlab-runner by: ~/packages/gitlab-runner/usr/bin/gitlab-runner 15 | 16 | # note: 'gitlab-runner exec' is currently rather limited :( 17 | # see: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2797 18 | # 19 | # Must create tweaked versions of ci .yml files to: 20 | # - use '<<' instead of 'extends' 21 | # - inline chgrp and chmod instead of using '*finalize' 22 | # - call each file and task directly: can't use 'include', 23 | # only runs explicitly listed task, one at a time 24 | # - seems to only include global vars and a final variables: declaration, doesn't merge intermediates from << 25 | # - seems to only see final script: declarations, doesn't merge intermediates from << 26 | # - sets pipefail, which breaks 'conda activate ...' 27 | # - [list all limitations...] 28 | 29 | # should we: 30 | # - source a conf file to set: SDK_DASHBOARD_{TOKEN,URL}, SDK_WORK_DIR, SITE 31 | # - create a full environment and run code from fresh clone/fetch each time for complete testing? 32 | # - set FF_ENABLE_JOB_CLEANUP ? https://docs.gitlab.com/runner/configuration/feature-flags.html 33 | # - cleanup /tmp/{old,new}-env-[0-9]+.txt 34 | # ? 35 | # Yes! Have too for scrontab and SDK_DASHBOARD_TOKEN atleast! 36 | # At a min, the rc file must be: 37 | # * executable 38 | # * and set the following (shown as example) 39 | #---start example rc 40 | ### should customize all of these for your site + installation: 41 | # export RUNNER=${RUNNER:-~/packages/gitlab-runner/usr/bin/gitlab-runner} 42 | # 43 | # export SDK_DASHBOARD_TOKEN=${SDK_DASHBOARD_TOKEN:-[token acquired from dashboard token request service]} 44 | # export SDK_DASHBOARD_URL=${SDK_DASHBOARD_URL:-https://sdk.testing.exaworks.org/result} 45 | # export SDK_WORK_DIR=${SDK_WORK_DIR:-/global/cfs/cdirs/m[####]/sdk} 46 | # 47 | # export SITE=${SITE:-perlmutter} 48 | # 49 | # export PIP_CONF_FILE=${PIP_CONF_FILE:-.gitlab/nersc-manual-gitlab-runner-pip.yml} 50 | # export CONDA_CONF_FILE=${CONDA_CONF_FILE:-.gitlab/nersc-manual-gitlab-runner-conda.yml} 51 | # 52 | # export CI_PIPELINE_ID=${RANDOM} 53 | # export CI_COMMIT_BRANCH="master" 54 | # export BUILDS_DIR="${SDK_WORK_DIR}/builds" 55 | #---end example rc 56 | 57 | echo "starting $0 run at $(date) in $(pwd) ..." 58 | 59 | export SITE_CI_GITLAB_RUNNER_SH_RC=${SITE_CI_GITLAB_RUNNER_SH_RC:-${HOME}/.nersc-ci-gitlab-runner.sh.rc} 60 | if [ -x "${SITE_CI_GITLAB_RUNNER_SH_RC}" ]; then 61 | . ${SITE_CI_GITLAB_RUNNER_SH_RC} 62 | fi 63 | 64 | if [ -z "$SDK_DASHBOARD_TOKEN" ]; then 65 | echo -e "$0:\tSDK_DASHBOARD_TOKEN env var must be set!!!\n" 66 | exit 67 | fi 68 | 69 | echo "cd'ing to \$REPO_DIR ($REPO_DIR)..." 70 | pushd $REPO_DIR 71 | 72 | ### for temp dev hacking... 73 | ###$RUNNER exec shell \ 74 | ### --env "SDK_DASHBOARD_TOKEN=${SDK_DASHBOARD_TOKEN}" \ 75 | ### --env "SDK_DASHBOARD_URL=${SDK_DASHBOARD_URL}" \ 76 | ### --env "SDK_WORK_DIR=${SDK_WORK_DIR}" \ 77 | ### --env "CI_PIPELINE_ID=${RANDOM}" \ 78 | ### --env 'CI_COMMIT_BRANCH=master' \ 79 | ### --builds-dir ${BUILDS_DIR} --cicd-config-file ${PIP_CONF_FILE} pip_env_setup_perlmutter 80 | 81 | #if [ "$1" == "pip" ]; then # tmp for debug 82 | for task in {pip_env_setup_,pip_build_,pip_test_}${SITE} pip_cleanup; do 83 | echo "### running: $task ..." 84 | date 85 | # hack to try to get cleanup working... give it a min to clear build dir from previous? 86 | if [ "$task" == "pip_cleanup" ]; then 87 | echo "sleeping for cleanup from previous..." 88 | sleep 180 89 | fi 90 | # echo "### checking build dir ..." 91 | # find ${BUILDS_DIR} -xdev -print0 | xargs -0r ls -altrd 92 | $RUNNER exec shell \ 93 | --env "SDK_DASHBOARD_TOKEN=${SDK_DASHBOARD_TOKEN}" \ 94 | --env "SDK_DASHBOARD_URL=${SDK_DASHBOARD_URL}" \ 95 | --env "SDK_WORK_DIR=${SDK_WORK_DIR}" \ 96 | --env "CI_PIPELINE_ID=${RANDOM}" \ 97 | --env 'CI_COMMIT_BRANCH=master' \ 98 | --builds-dir ${BUILDS_DIR} --cicd-config-file ${PIP_CONF_FILE} $task 99 | done 100 | #else # tmp for debug 101 | for task in {conda_setup_,conda_env_setup_,conda_build_,conda_test_}${SITE} conda_cleanup; do 102 | echo "### running: $task ..." 103 | date 104 | # hack to try to get cleanup working... give it a min to clear build dir from previous? 105 | if [ "$task" == "conda_cleanup" ]; then 106 | echo "sleeping for cleanup from previous..." 107 | sleep 180 108 | fi 109 | # echo "### checking build dir ..." 110 | # find ${BUILDS_DIR} -xdev -print0 | xargs -0r ls -altrd 111 | $RUNNER exec shell \ 112 | --env "SDK_DASHBOARD_TOKEN=${SDK_DASHBOARD_TOKEN}" \ 113 | --env "SDK_DASHBOARD_URL=${SDK_DASHBOARD_URL}" \ 114 | --env "SDK_WORK_DIR=${SDK_WORK_DIR}" \ 115 | --env "CI_PIPELINE_ID=${RANDOM}" \ 116 | --env 'CI_COMMIT_BRANCH=master' \ 117 | --builds-dir ${BUILDS_DIR} --cicd-config-file ${CONDA_CONF_FILE} $task 118 | done 119 | #fi # tmp for debug 120 | date 121 | -------------------------------------------------------------------------------- /.gitlab/nersc-ci-pip.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | PIP_WORK_DIR: ${SDK_WORK_DIR}/pip 3 | PIP_REQUIREMENTS: .gitlab/pip-requirements.txt 4 | 5 | 6 | stages: 7 | - setup 8 | - build 9 | - test 10 | - cleanup 11 | 12 | 13 | ### MACHINES 14 | .on_perlmutter: 15 | variables: 16 | SITE: "perlmutter" 17 | VENV_ENV_NAME: "perlmutter-env" 18 | # python/3.10 works too, python/3.11 breaks on 'pip install --no-cache-dir -r ${PIP_REQUIREMENTS}' in .pip_build; fails to satisfy requirement smartsim==0.5.1 19 | PYTHON_MODULE: "python/3.9" 20 | 21 | 22 | ### SCRIPTS 23 | .final_steps: 24 | script: &finalize 25 | - chgrp -fR m3973 ${PIP_WORK_DIR} || true 26 | - chmod -fR 02770 ${PIP_WORK_DIR} || true 27 | 28 | .job_tags: 29 | tags: 30 | - $SITE 31 | - $RUNNER_TYPE 32 | 33 | .pip_env_setup: 34 | stage: setup 35 | script: 36 | - mkdir -p ${PIP_WORK_DIR} 37 | - test -d ${PIP_WORK_DIR}/${VENV_ENV_NAME} && exit 0 38 | - module add ${PYTHON_MODULE} 39 | - python3 -m venv ${PIP_WORK_DIR}/${VENV_ENV_NAME} 40 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 41 | - pip install -U pip setuptools wheel 42 | - pip cache purge 43 | - *finalize 44 | 45 | .pip_build: 46 | stage: build 47 | script: 48 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 49 | - pip install --no-cache-dir -r ${PIP_REQUIREMENTS} 50 | - (smart clean && smart build) > /dev/null 2>&1 || true 51 | - *finalize 52 | 53 | .pip_test: 54 | stage: test 55 | variables: 56 | MAINTAINER: "cowan@bnl.gov" 57 | IM_NUMBER: "" 58 | TESTS_GROUP: "pip" 59 | SITE_ID: "nersc-$SITE" 60 | script: 61 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 62 | - python3 ci/tests/test.py -s 63 | - for TEST in parsl rp maestro smartsim; do 64 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 65 | done 66 | - python3 ci/tests/test.py -e 67 | - deactivate 68 | 69 | ### JOBS 70 | 71 | # PERLMUTTER 72 | pip_env_setup_perlmutter: 73 | variables: 74 | RUNNER_TYPE: "shell" 75 | extends: [ .on_perlmutter, .job_tags, .pip_env_setup ] 76 | 77 | pip_build_perlmutter: 78 | variables: 79 | RUNNER_TYPE: "shell" 80 | needs: [ pip_env_setup_perlmutter ] 81 | extends: [ .on_perlmutter, .job_tags, .pip_build ] 82 | 83 | pip_test_perlmutter: 84 | variables: 85 | RUNNER_TYPE: "batch" 86 | SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 87 | needs: [ pip_build_perlmutter ] 88 | extends: [ .on_perlmutter, .job_tags, .pip_test ] 89 | 90 | 91 | # ALWAYS 92 | pip_cleanup: 93 | stage: cleanup 94 | when: always 95 | variables: 96 | RUNNER_TYPE: "shell" 97 | extends: [ .on_perlmutter, .job_tags ] 98 | script: 99 | - *finalize 100 | 101 | -------------------------------------------------------------------------------- /.gitlab/nersc-ci-scrontab.example: -------------------------------------------------------------------------------- 1 | # Wanted to determine 2 sets of resources needed: 2 | # - presumably more resources helpful for initial ci run? 3 | # - and less needed for subsequent runs reusing envs built by initial run? 4 | # but didn't quite get that all ironed out. 5 | # 6 | # At a minimum, would need to replace these below: 7 | # - [nersc_project] 8 | # - [full_path_for_output_log] 9 | # - [full_path_for_SDK_repo] 10 | # 11 | #SCRON -q shared 12 | # for full runs with complete env setup and build... 13 | # '-n 5 --ntasks-per-core=1' seemed exceedingly slow for even checking out source!? 14 | ##SCRON -n 5 15 | #SCRON -n 10 16 | #SCRON --ntasks-per-core=1 17 | #SCRON --mem=5G 18 | # 60 seems to not be quite enough in some cases?! 19 | #SCRON -t 90 20 | ### for followup runs reusing previously built env... (haven't quite fully optimized this yet...) 21 | ###SCRON -n 2 22 | ###SCRON --ntasks-per-core=5 23 | ###SCRON --mem=2G 24 | ###SCRON -t 30 25 | # 26 | #SCRON -C cpu 27 | #SCRON -A [nersc_project] 28 | # full run from scratch = ~21m conda + ~34m pip =~ 60m 29 | # followup run = ~9m conda + ~19m pip =~ 30m 30 | #SCRON --dependency=singleton 31 | #SCRON -o [full_path_for_output_log]/scron-%j.log 32 | #SCRON --open-mode=append 33 | 30 5 * * * [full_path_for_SDK_repo]/.gitlab/nersc-ci-gitlab-runner.sh 34 | -------------------------------------------------------------------------------- /.gitlab/nersc-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - local: '.gitlab/nersc-ci-*.yml' 3 | 4 | stages: 5 | - setup 6 | - build 7 | - test 8 | - cleanup 9 | -------------------------------------------------------------------------------- /.gitlab/nersc-manual-gitlab-runner-conda.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | CONDA_WORK_DIR: ${SDK_WORK_DIR}/conda 3 | CONDA_REQUIREMENTS: .gitlab/conda-requirements.yml 4 | SITE: "perlmutter" 5 | VENV_ENV_NAME: "perlmutter-env" 6 | # python/3.10 works too, python/3.11 breaks on 'pip install --no-cache-dir -r ${PIP_REQUIREMENTS}' in .pip_build; fails to satisfy requirement smartsim==0.5.1 7 | PYTHON_MODULE: "python/3.9" 8 | CONDA_ENV_NAME: "perlmutter-env" 9 | CONDA_ARCH: "x86_64" 10 | MAINTAINER: "cowan@bnl.gov" 11 | IM_NUMBER: "" 12 | TESTS_GROUP: "conda" 13 | SITE_ID: "nersc-$SITE" 14 | 15 | 16 | stages: 17 | - setup 18 | - build 19 | - test 20 | - cleanup 21 | 22 | 23 | ### MACHINES 24 | # a limitation of 'gitlab-runner exec shell' seems to be only global and last 'variables:' 25 | # and 'script:' sections work. Doesn't correctly merge intermediate declarations from << :( 26 | #.on_perlmutter: &on_perlmutter_tmpl 27 | # variables: 28 | # NAME: "value" 29 | 30 | ### SCRIPTS 31 | .job_tags: &job_tags_tmpl 32 | tags: 33 | - $SITE 34 | - $RUNNER_TYPE 35 | 36 | .conda_setup: &conda_setup_tmpl 37 | stage: setup 38 | script: 39 | - mkdir -p ${CONDA_WORK_DIR} 40 | - test -d ${CONDA_WORK_DIR}/miniconda3-${SITE} && exit 0 41 | - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 42 | - chmod +x ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 43 | - ${CONDA_WORK_DIR}/miniconda-${SITE}.sh -b -p ${CONDA_WORK_DIR}/miniconda3-${SITE} 44 | # - chgrp -fR m3973 ${PIP_WORK_DIR} || true 45 | # - chmod -fR 02770 ${PIP_WORK_DIR} || true 46 | 47 | .conda_env_setup: &conda_env_setup_tmpl 48 | stage: setup 49 | script: 50 | - test -d ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} && exit 0 51 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 52 | # confirm 'gitlab-runner exec shell ...' is setting pipefail 53 | - set -o | grep pipe 54 | # disable pipefail because it causes eval and activate below to fail 55 | - set +o pipefail 56 | - eval "$(conda shell.posix hook)" 57 | - conda create --prefix ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} -y 58 | # - chgrp -fR m3973 ${PIP_WORK_DIR} || true 59 | # - chmod -fR 02770 ${PIP_WORK_DIR} || true 60 | 61 | .conda_build: &conda_build_tmpl 62 | stage: build 63 | script: 64 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 65 | # confirm 'gitlab-runner exec shell ...' is setting pipefail 66 | - set -o | grep pipe 67 | # disable pipefail because it causes eval and activate below to fail 68 | - set +o pipefail 69 | - eval "$(conda shell.posix hook)" 70 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 71 | - conda env update --file ${CONDA_REQUIREMENTS} 72 | # - chgrp -fR m3973 ${PIP_WORK_DIR} || true 73 | # - chmod -fR 02770 ${PIP_WORK_DIR} || true 74 | 75 | .conda_test: &conda_test_tmpl 76 | stage: test 77 | script: 78 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 79 | # confirm 'gitlab-runner exec shell ...' is setting pipefail 80 | - set -o | grep pipe 81 | # disable pipefail because it causes eval and activate below to fail 82 | - set +o pipefail 83 | - eval "$(conda shell.posix hook)" 84 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 85 | - python3 ci/tests/test.py -s 86 | - for TEST in parsl rp swift-t; do 87 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 88 | done 89 | - python3 ci/tests/test.py -e 90 | # - chgrp -fR m3973 ${PIP_WORK_DIR} || true 91 | # - chmod -fR 02770 ${PIP_WORK_DIR} || true 92 | 93 | 94 | ### JOBS 95 | 96 | # PERLMUTTER 97 | conda_setup_perlmutter: 98 | variables: 99 | RUNNER_TYPE: "shell" 100 | # <<: *on_perlmutter_tmpl 101 | <<: *job_tags_tmpl 102 | <<: *conda_setup_tmpl 103 | 104 | conda_env_setup_perlmutter: 105 | variables: 106 | RUNNER_TYPE: "shell" 107 | needs: [ conda_setup_perlmutter ] 108 | # <<: *on_perlmutter_tmpl 109 | <<: *job_tags_tmpl 110 | <<: *conda_env_setup_tmpl 111 | 112 | conda_build_perlmutter: 113 | variables: 114 | RUNNER_TYPE: "shell" 115 | needs: [ conda_env_setup_perlmutter ] 116 | # <<: *on_perlmutter_tmpl 117 | <<: *job_tags_tmpl 118 | <<: *conda_build_tmpl 119 | 120 | conda_test_perlmutter: 121 | variables: 122 | RUNNER_TYPE: "batch" 123 | SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 124 | needs: [ conda_build_perlmutter ] 125 | # <<: *on_perlmutter_tmpl 126 | <<: *job_tags_tmpl 127 | <<: *conda_test_tmpl 128 | 129 | 130 | # ALWAYS 131 | conda_cleanup: 132 | stage: cleanup 133 | when: always 134 | variables: 135 | RUNNER_TYPE: "shell" 136 | # <<: *on_perlmutter_tmpl 137 | <<: *job_tags_tmpl 138 | script: 139 | - time chgrp -fR m3973 ${CONDA_WORK_DIR} || true 140 | - time chmod -fR 02770 ${CONDA_WORK_DIR} || true 141 | -------------------------------------------------------------------------------- /.gitlab/nersc-manual-gitlab-runner-pip.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | PIP_WORK_DIR: ${SDK_WORK_DIR}/pip 3 | PIP_REQUIREMENTS: .gitlab/pip-requirements.txt 4 | SITE: "perlmutter" 5 | VENV_ENV_NAME: "perlmutter-env" 6 | # python/3.10 works too, python/3.11 breaks on 'pip install --no-cache-dir -r ${PIP_REQUIREMENTS}' in .pip_build; fails to satisfy requirement smartsim==0.5.1 7 | PYTHON_MODULE: "python/3.9" 8 | MAINTAINER: "cowan@bnl.gov" 9 | IM_NUMBER: "" 10 | TESTS_GROUP: "pip" 11 | SITE_ID: "nersc-$SITE" 12 | 13 | 14 | stages: 15 | - setup 16 | - build 17 | - test 18 | - cleanup 19 | 20 | 21 | ### MACHINES 22 | #.on_perlmutter: &on_perlmutter_tmpl 23 | # variables: 24 | 25 | ### SCRIPTS 26 | .job_tags: &job_tags_tmpl 27 | tags: 28 | - $SITE 29 | - $RUNNER_TYPE 30 | 31 | .pip_env_setup: &pip_env_setup_tmpl 32 | stage: setup 33 | script: 34 | - mkdir -p ${PIP_WORK_DIR} 35 | - test -d ${PIP_WORK_DIR}/${VENV_ENV_NAME} && exit 0 36 | - module add ${PYTHON_MODULE} 37 | - python3 -m venv ${PIP_WORK_DIR}/${VENV_ENV_NAME} 38 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 39 | - pip install --upgrade pip 40 | - pip install -U pip setuptools wheel 41 | - pip cache purge 42 | # - chgrp -fR m3973 ${PIP_WORK_DIR} || true 43 | # - chmod -fR 02770 ${PIP_WORK_DIR} || true 44 | 45 | .pip_build: &pip_build_tmpl 46 | stage: build 47 | script: 48 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 49 | - pip install --no-cache-dir -r ${PIP_REQUIREMENTS} 50 | - (smart clean && smart build) > /dev/null 2>&1 || true 51 | # - chgrp -fR m3973 ${PIP_WORK_DIR} || true 52 | # - chmod -fR 02770 ${PIP_WORK_DIR} || true 53 | 54 | 55 | .pip_test: &pip_test_tmpl 56 | stage: test 57 | script: 58 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 59 | - python3 ci/tests/test.py -s 60 | - for TEST in parsl rp maestro smartsim; do 61 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 62 | done 63 | - python3 ci/tests/test.py -e 64 | - deactivate 65 | # - chgrp -fR m3973 ${PIP_WORK_DIR} || true 66 | # - chmod -fR 02770 ${PIP_WORK_DIR} || true 67 | 68 | 69 | ### JOBS 70 | 71 | # PERLMUTTER 72 | pip_env_setup_perlmutter: 73 | variables: 74 | RUNNER_TYPE: "shell" 75 | # <<: *on_perlmutter_tmpl 76 | <<: *job_tags_tmpl 77 | <<: *pip_env_setup_tmpl 78 | 79 | pip_build_perlmutter: 80 | variables: 81 | RUNNER_TYPE: "shell" 82 | needs: [ pip_env_setup_perlmutter ] 83 | # <<: *on_perlmutter_tmpl 84 | <<: *job_tags_tmpl 85 | <<: *pip_build_tmpl 86 | 87 | pip_test_perlmutter: 88 | variables: 89 | RUNNER_TYPE: "batch" 90 | SCHEDULER_PARAMETERS: "--nodes=1 -t 45" 91 | needs: [ pip_build_perlmutter ] 92 | # <<: *on_perlmutter_tmpl 93 | <<: *job_tags_tmpl 94 | <<: *pip_test_tmpl 95 | 96 | 97 | # ALWAYS 98 | pip_cleanup: 99 | stage: cleanup 100 | when: always 101 | variables: 102 | RUNNER_TYPE: "shell" 103 | # <<: *on_perlmutter_tmpl 104 | <<: *job_tags_tmpl 105 | script: 106 | - time chgrp -fR m3973 ${PIP_WORK_DIR} || true 107 | - time chmod -fR 02770 ${PIP_WORK_DIR} || true 108 | -------------------------------------------------------------------------------- /.gitlab/ornl-ci-conda.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - setup 3 | - build 4 | - test 5 | - cleanup 6 | 7 | 8 | ### MACHINES 9 | .on_ascent: 10 | variables: 11 | SITE: "ascent" 12 | CONDA_ENV_NAME: "ascent-env" 13 | CONDA_WORK_DIR: ${SDK_WORK_DIR_OPEN}/conda 14 | CONDA_REQUIREMENTS: .gitlab/conda-requirements-ppc64le.yml 15 | PYTHON_MODULE: "python/3.8-anaconda3" 16 | CONDA_ARCH: "ppc64le" 17 | 18 | # .on_summit: 19 | # variables: 20 | # SITE: "summit" 21 | # CONDA_ENV_NAME: "summit-env" 22 | # CONDA_WORK_DIR: ${SDK_WORK_DIR_ALPINE}/conda 23 | # CONDA_REQUIREMENTS: .gitlab/conda-requirements-ppc64le.yml 24 | # PYTHON_MODULE: "python/3.8-anaconda3" 25 | # CONDA_ARCH: "ppc64le" 26 | 27 | 28 | ### SCRIPTS 29 | .final_steps: 30 | script: &finalize 31 | - chgrp -fR csc449 ${CONDA_WORK_DIR} || true 32 | - chmod -fR 02770 ${CONDA_WORK_DIR} || true 33 | 34 | .job_tags: 35 | tags: 36 | - $SITE 37 | - $RUNNER_TYPE 38 | 39 | .conda_setup: 40 | stage: setup 41 | script: 42 | - mkdir -p ${CONDA_WORK_DIR} 43 | - test -d ${CONDA_WORK_DIR}/miniconda3-${SITE} && exit 0 44 | - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 45 | - chmod +x ${CONDA_WORK_DIR}/miniconda-${SITE}.sh 46 | - ${CONDA_WORK_DIR}/miniconda-${SITE}.sh -b -p ${CONDA_WORK_DIR}/miniconda3-${SITE} 47 | - *finalize 48 | 49 | .conda_env_setup: 50 | stage: setup 51 | script: 52 | - test -d ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} && exit 0 53 | - module add ${PYTHON_MODULE} 54 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 55 | - eval "$(conda shell.posix hook)" 56 | - conda create --prefix ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} -y 57 | - *finalize 58 | 59 | .conda_build: 60 | stage: build 61 | script: 62 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 63 | - eval "$(conda shell.posix hook)" 64 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 65 | - conda env update --file ${CONDA_REQUIREMENTS} 66 | - *finalize 67 | 68 | .conda_test: 69 | stage: test 70 | variables: 71 | MAINTAINER: "arambula2@llnl.gov" 72 | IM_NUMBER: "" 73 | TESTS_GROUP: "conda" 74 | SITE_ID: "ornl-$SITE" 75 | script: 76 | - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh 77 | - eval "$(conda shell.posix hook)" 78 | - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} 79 | - python3 ci/tests/test.py -s 80 | - for TEST in parsl rp; do 81 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 82 | done 83 | - python3 ci/tests/test.py -e 84 | - *finalize 85 | 86 | .conda_cleanup: 87 | stage: cleanup 88 | when: always 89 | script: 90 | - find ${CONDA_WORK_DIR} -maxdepth 0 -type d -ctime +5 | xargs rm -rf 91 | - *finalize 92 | 93 | 94 | ### JOBS 95 | 96 | # ASCENT 97 | conda_setup_ascent: 98 | variables: 99 | RUNNER_TYPE: "nobatch" 100 | extends: [ .on_ascent, .job_tags, .conda_setup ] 101 | 102 | conda_env_setup_ascent: 103 | variables: 104 | RUNNER_TYPE: "nobatch" 105 | needs: [ conda_setup_ascent ] 106 | extends: [ .on_ascent, .job_tags, .conda_env_setup ] 107 | 108 | conda_build_ascent: 109 | variables: 110 | RUNNER_TYPE: "nobatch" 111 | needs: [ conda_env_setup_ascent ] 112 | extends: [ .on_ascent, .job_tags, .conda_build ] 113 | 114 | conda_test_ascent: 115 | variables: 116 | RUNNER_TYPE: "batch" 117 | SCHEDULER_PARAMETERS: "-P CSC449 -nnodes 1 -W 45" 118 | needs: [ conda_build_ascent ] 119 | extends: [ .on_ascent, .job_tags, .conda_test ] 120 | 121 | 122 | # ALWAYS 123 | conda_cleanup_ascent: 124 | variables: 125 | RUNNER_TYPE: "nobatch" 126 | extends: [ .on_ascent, .job_tags, .conda_cleanup ] 127 | 128 | -------------------------------------------------------------------------------- /.gitlab/ornl-ci-pip.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | PIP_REQUIREMENTS: .gitlab/pip-requirements.txt 3 | 4 | 5 | stages: 6 | - setup 7 | - build 8 | - test 9 | - cleanup 10 | 11 | 12 | ### MACHINES 13 | .on_ascent: 14 | variables: 15 | SITE: "ascent" 16 | VENV_ENV_NAME: "ascent-env" 17 | PIP_WORK_DIR: ${SDK_WORK_DIR_OPEN}/pip 18 | PYTHON_MODULE: "python/3.8-anaconda3" 19 | 20 | # .on_summit: 21 | # variables: 22 | # SITE: "summit" 23 | # VENV_ENV_NAME: "summit-env" 24 | # PIP_WORK_DIR: ${SDK_WORK_DIR_ALPINE}/pip 25 | # PYTHON_MODULE: "python/3.8-anaconda3" 26 | 27 | 28 | ### SCRIPTS 29 | .final_steps: 30 | script: &finalize 31 | - chgrp -fR csc449 ${PIP_WORK_DIR} || true 32 | - chmod -fR 02770 ${PIP_WORK_DIR} || true 33 | 34 | .job_tags: 35 | tags: 36 | - $SITE 37 | - $RUNNER_TYPE 38 | 39 | .pip_env_setup: 40 | stage: setup 41 | script: 42 | - mkdir -p ${PIP_WORK_DIR} 43 | - test -d ${PIP_WORK_DIR}/${VENV_ENV_NAME} && exit 0 44 | - module add ${PYTHON_MODULE} 45 | - python3 -m venv ${PIP_WORK_DIR}/${VENV_ENV_NAME} 46 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 47 | - pip install -U pip setuptools wheel 48 | - pip cache purge 49 | - *finalize 50 | 51 | .pip_build: 52 | stage: build 53 | script: 54 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 55 | - pip install --no-cache-dir -r ${PIP_REQUIREMENTS} 56 | - *finalize 57 | 58 | .pip_test: 59 | stage: test 60 | variables: 61 | MAINTAINER: "arambula2@llnl.gov" 62 | IM_NUMBER: "" 63 | TESTS_GROUP: "pip" 64 | SITE_ID: "ornl-$SITE" 65 | script: 66 | - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate 67 | - python3 ci/tests/test.py -s 68 | - for TEST in flux parsl rp parsl-flux rp-flux maestro; do 69 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 70 | done 71 | - python3 ci/tests/test.py -e 72 | - deactivate 73 | 74 | .pip_cleanup: 75 | stage: cleanup 76 | when: always 77 | script: 78 | - find ${PIP_WORK_DIR} -maxdepth 0 -type d -ctime +5 | xargs rm -rf 79 | - *finalize 80 | 81 | ### JOBS 82 | 83 | # ascent 84 | pip_env_setup_ascent: 85 | variables: 86 | RUNNER_TYPE: "nobatch" 87 | extends: [ .on_ascent, .job_tags, .pip_env_setup ] 88 | 89 | pip_build_ascent: 90 | variables: 91 | RUNNER_TYPE: "nobatch" 92 | needs: [ pip_env_setup_ascent ] 93 | extends: [ .on_ascent, .job_tags, .pip_build ] 94 | 95 | pip_test_ascent: 96 | variables: 97 | RUNNER_TYPE: "batch" 98 | SCHEDULER_PARAMETERS: "-P CSC449 -nnodes 1 -W 45" 99 | needs: [ pip_build_ascent ] 100 | extends: [ .on_ascent, .job_tags, .pip_test ] 101 | 102 | pip_cleanup_ascent: 103 | variables: 104 | RUNNER_TYPE: "nobatch" 105 | extends: [ .on_ascent, .job_tags, .pip_cleanup ] 106 | 107 | -------------------------------------------------------------------------------- /.gitlab/ornl-ci-spack.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - setup 3 | - build 4 | - test 5 | - cleanup 6 | 7 | 8 | ### MACHINES 9 | .on_ascent: 10 | variables: 11 | SITE: "ascent" 12 | SPACK_ENV_NAME: "ascent-env" 13 | SPACK_WORK_DIR: ${SDK_WORK_DIR_OPEN}/spack 14 | 15 | # .on_summit: 16 | # variables: 17 | # SITE: "summit" 18 | # SPACK_ENV_NAME: "summit-env" 19 | # SPACK_WORK_DIR: ${SDK_WORK_DIR_ALPINE}/spack 20 | 21 | 22 | ### SCRIPTS 23 | .final_steps: 24 | script: &finalize 25 | - chgrp -fR csc449 ${SPACK_WORK_DIR} || true 26 | - chmod -fR 02770 ${SPACK_WORK_DIR} || true 27 | 28 | .job_tags: 29 | tags: 30 | - $SITE 31 | - $RUNNER_TYPE 32 | 33 | .spack_env_setup: 34 | stage: setup 35 | script: 36 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 37 | - test "$(spack env list | grep $SPACK_ENV_NAME)" && exit 0 38 | - spack env create ${SPACK_ENV_NAME} 39 | - for i in {1..3}; do 40 | spack env activate ${SPACK_ENV_NAME} && break || sleep 5; 41 | done 42 | - spack config add concretizer:unify:when_possible 43 | - spack config add concretizer:reuse:false 44 | - spack config add config:db_lock_timeout:300 45 | - module --latest load gcc && spack compiler find 46 | - spack add rust ^openssl certs=system 47 | - spack install || (spack env deactivate && spack env remove ${SPACK_ENV_NAME} -y && exit 1) 48 | - *finalize 49 | 50 | .spack_build: 51 | stage: build 52 | script: 53 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 54 | - for i in {1..3}; do 55 | spack env activate ${SPACK_ENV_NAME} && break || sleep 5; 56 | done 57 | - spack uninstall -y exaworks py-pytest || true 58 | - spack add exaworks ^python@3.9 py-pytest 59 | - spack concretize -f 60 | - spack install 61 | - *finalize 62 | 63 | .spack_test: 64 | stage: test 65 | variables: 66 | MAINTAINER: "arambula2@llnl.gov" 67 | IM_NUMBER: "" 68 | TESTS_GROUP: "spack" 69 | SITE_ID: "ornl-$SITE" 70 | script: 71 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 72 | - for i in {1..3}; do 73 | spack env activate ${SPACK_ENV_NAME} && break || sleep 5; 74 | done 75 | - spack load exaworks py-pytest 76 | - python3 ci/tests/test.py -s 77 | - for TEST in flux parsl rp swift-t parsl-flux rp-flux; do 78 | python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout; 79 | done 80 | - python3 ci/tests/test.py -e 81 | 82 | ### JOBS 83 | 84 | spack_setup: 85 | stage: setup 86 | variables: 87 | RUNNER_TYPE: "nobatch" 88 | extends: [ .on_ascent, .job_tags ] 89 | script: 90 | - test -d ${SPACK_WORK_DIR} && exit 0 91 | - git clone -c feature.manyFiles=true -c core.sharedRepository=true https://github.com/spack/spack.git ${SPACK_WORK_DIR} 92 | - *finalize 93 | 94 | spack_update: 95 | stage: build 96 | variables: 97 | RUNNER_TYPE: "nobatch" 98 | extends: [ .on_ascent, .job_tags ] 99 | needs: [ spack_env_setup_ascent ] 100 | script: 101 | - cd ${SPACK_WORK_DIR} 102 | - git checkout HEAD^ . 103 | - git reset --hard HEAD 104 | - git pull --ff 105 | - *finalize 106 | 107 | 108 | # ASCENT 109 | spack_env_setup_ascent: 110 | variables: 111 | RUNNER_TYPE: "nobatch" 112 | needs: [ spack_setup ] 113 | extends: [ .on_ascent, .job_tags, .spack_env_setup ] 114 | 115 | spack_build_ascent: 116 | variables: 117 | RUNNER_TYPE: "nobatch" 118 | needs: [ spack_update ] 119 | extends: [ .on_ascent, .job_tags, .spack_build ] 120 | 121 | spack_test_ascent: 122 | variables: 123 | RUNNER_TYPE: "batch" 124 | SCHEDULER_PARAMETERS: "-P CSC449 -nnodes 1 -W 45" 125 | needs: [ spack_build_ascent ] 126 | extends: [ .on_ascent, .job_tags, .spack_test ] 127 | 128 | 129 | # ALWAYS 130 | spack_cleanup: 131 | stage: cleanup 132 | when: always 133 | variables: 134 | RUNNER_TYPE: "nobatch" 135 | extends: [ .on_ascent, .job_tags ] 136 | script: 137 | - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh 138 | - spack clean -a || true 139 | - rm -rf ${HOME}/.spack/cache 140 | - spack providers 141 | - find ${SPACK_WORK_DIR} -maxdepth 0 -type d -ctime +5 | xargs rm -rf 142 | - *finalize 143 | 144 | -------------------------------------------------------------------------------- /.gitlab/ornl-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - local: '.gitlab/ornl-ci-*.yml' 3 | 4 | stages: 5 | - setup 6 | - build 7 | - test 8 | - cleanup 9 | -------------------------------------------------------------------------------- /.gitlab/pip-requirements.txt: -------------------------------------------------------------------------------- 1 | # parsl dependencies to avoid rust installation 2 | bcrypt<4.0.0 3 | cryptography==3.3.2 4 | globus_sdk<3.1 5 | 6 | # core components 7 | parsl 8 | radical.pilot 9 | maestrowf 10 | 11 | smartsim==0.5.1; python_version>="3.8" and platform_machine=="x86_64" 12 | tensorflow==2.8.0; python_version>="3.8" and platform_machine=="x86_64" 13 | 14 | # tests 15 | pytest 16 | pytest-tap 17 | 18 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | queue_rules: 2 | - name: default 3 | conditions: &ci_checks 4 | # Conditions to get out of the queue (= merged) 5 | - status-success="Run test suite (centos7, pip, openmpi, 3.7)" 6 | - status-success="Run test suite (centos7, pip, openmpi, 3.8)" 7 | - status-success="Run test suite (centos7, pip, openmpi, 3.9)" 8 | - status-success="Run test suite (centos7, pip, mpich, 3.7)" 9 | - status-success="Run test suite (centos7, pip, mpich, 3.8)" 10 | - status-success="Run test suite (centos7, pip, mpich, 3.9)" 11 | - status-success="Run test suite (rockylinux8, pip, openmpi, 3.7)" 12 | - status-success="Run test suite (rockylinux8, pip, openmpi, 3.8)" 13 | - status-success="Run test suite (rockylinux8, pip, openmpi, 3.9)" 14 | - status-success="Run test suite (rockylinux8, pip, mpich, 3.7)" 15 | - status-success="Run test suite (rockylinux8, pip, mpich, 3.8)" 16 | - status-success="Run test suite (rockylinux8, pip, mpich, 3.9)" 17 | - status-success="Run test suite (ubuntu2004, pip, openmpi, 3.7)" 18 | - status-success="Run test suite (ubuntu2004, pip, openmpi, 3.8)" 19 | - status-success="Run test suite (ubuntu2004, pip, openmpi, 3.9)" 20 | - status-success="Run test suite (ubuntu2004, pip, mpich, 3.7)" 21 | - status-success="Run test suite (ubuntu2004, pip, mpich, 3.8)" 22 | - status-success="Run test suite (ubuntu2004, pip, mpich, 3.9)" 23 | - status-success="validate commits" 24 | 25 | pull_request_rules: 26 | - name: Automatic merge on approval 27 | conditions: 28 | # Conditions to get into the queue (= queued for merge) 29 | - and: *ci_checks 30 | - base=master 31 | - label="merge-when-passing" 32 | - label!="work-in-progress" 33 | - "#approved-reviews-by>=1" 34 | - "#changes-requested-reviews-by=0" 35 | - -title~=^\[*[Ww][Ii][Pp] 36 | actions: 37 | queue: 38 | method: rebase 39 | name: default 40 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: "ubuntu-22.04" 5 | tools: 6 | python: "3.7" 7 | apt_packages: 8 | - libopenmpi-dev 9 | - openmpi-bin 10 | - openjdk-11-jdk 11 | - ant 12 | - mpich 13 | - swig 14 | - zsh 15 | - nwchem 16 | jobs: 17 | post_install: 18 | - wget http://swift-lang.github.io/swift-t-downloads/1.6/swift-t-1.6.0.tar.gz 19 | - tar xfz swift-t-1.6.0.tar.gz 20 | - swift-t-1.6.0/dev/build/init-settings.sh 21 | - swift-t-1.6.0/dev/build/build-swift-t.sh 22 | 23 | formats: all 24 | 25 | python: 26 | install: 27 | - requirements: docs/source/requirements.txt 28 | system_packages: true 29 | 30 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution Guide 2 | 3 | This project welcomes all contributors. This short guide (based loosely on [ 4 | Collective Code Construction Contract](http://zeromq-rfc.wikidot.com/spec:22) 5 | and [matplotlib's development 6 | workflow](https://matplotlib.org/stable/devel/gitwash/development_workflow.html#development-workflow)) 7 | details how to contribute in a standardized and efficient manner. 8 | 9 | ## Git Workflow Summary 10 | 11 | - Ensure that you've opened an Issue on Github and consensus around the 12 | solution has be reached. 13 | - Minor changes (e.g., grammatical fixes) do not require an Issue first. 14 | - [Create your own 15 | fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo). 16 | - When you are starting a new set of changes, [fetch any changes from the 17 | upstream 18 | repo](https://matplotlib.org/stable/devel/gitwash/development_workflow.html#update-the-mirror-of-trunk), 19 | and [start a new feature branch on your fork from 20 | that](https://matplotlib.org/stable/devel/gitwash/development_workflow.html#make-a-new-feature-branch). 21 | - Make a new branch for each separable set of changes — ["one task, one 22 | branch"](https://mail.python.org/pipermail/ipython-dev/2010-October/005632.html) 23 | - [Each commit should make one change](https://dev.to/ruanbrandao/how-to-make-good-git-commits-256k). 24 | to aide reviewing and (in the worst case) simplify reverting it in the future. 25 | - A patch commit message should consist of a single short (less than 50 26 | character) line summarizing the change, optionally followed by a blank line 27 | and then a more thorough description. 28 | - Where applicable, a PR or commit message body should reference an Issue by 29 | number (e.g. `Fixes #33`). 30 | - If you can possibly avoid it, avoid merging upstream branches or any other 31 | branches into your feature branch while you are working. 32 | - If you do find yourself merging from upstream, consider [Rebasing on 33 | upstream](https://matplotlib.org/stable/devel/gitwash/development_workflow.html#rebase-on-trunk). 34 | - Submit a Pull Request from your feature branch against upstream. 35 | - Use the Draft PR feature on Github or title your PR with `WIP` if your PR is 36 | not ready for a complete review immediately upon submission. 37 | - Ask on the [Exaworks slack](https://exaworks.slack.com) if you get stuck. 38 | 39 | 40 | ## Pull Request (PR) Merging Process 41 | 42 | - PR reviews should be timely. Both reviewer and PR issuer should make a good 43 | attempt at resolving the conversation as quickly as possible. 44 | - PR reviews exist to check obvious things aren't missed, not to achieve 45 | perfection. 46 | - A PR is eligible for merging if it has at least one approval from a 47 | project maintainer, no outstanding requested changes or discussions, and passes 48 | CI tests (if configured for the repository). 49 | - Discussions created via an inline comment on GitHub should only be "resolved" 50 | by whomever opened the discussion. 51 | - The person to mark the last open discussion "resolved" should also merge the 52 | PR ("close the door when you leave"), unless a merge bot is configured for the 53 | repository, in which case the PR should be left for the bot to merge. 54 | - Maintainers should not merge their own patches except in exceptional cases. 55 | -------------------------------------------------------------------------------- /POLICIES.md: -------------------------------------------------------------------------------- 1 | # Introduction and Goals 2 | The ExaWorks SDK aims to support the workflow needs of a diverse range of users, administrators, and developers. It will enable teams to produce scalable and portable workflows for a wide range of exascale applications. The SDK will not replace the many workflow solutions already deployed and used by scientists, but rather it will provide a robust collection of community-identified technologies and components that can be leveraged by users. Most importantly, the SDK will enable a sustainable software infrastructure for workflows so that the software artifacts produced by teams will be easier to port, modify, and utilize long after projects end. 3 | 4 | This document outlines the policies and recommendations for inclusion in the ExaWorks SDK. The policies are based on those defined by the xSDK and LLNL RADIUSS projects and reflect best practices for open source development, development of sustainable software, and scalable deployment of software on ECP systems. 5 | 6 | * M indicates a mandatory policy 7 | * R indicates a recommended policy 8 | 9 | ## Licensing 10 | M: Use an OSI-approved open-source license (e.g., Apache, MIT, BSD, LGPL) 11 | 12 | R: Provide a list of dependencies and their licenses in a standard format (e.g., SPDX) 13 | 14 | ## Code 15 | M: Code should be version controlled and publicly accessible online 16 | 17 | M: Provide a transparent, online contribution process based on published contributing guide, and using pull requests and issues collection 18 | 19 | R: Code should be version controlled using Git and accessible on GitHub 20 | 21 | R: Follow a common style guide for code layout, documentation, naming, etc. (e.g., PEP8) 22 | 23 | R: Code should be documented in a format that can be automatically parsed. E.g., Python's Sphinx Style, Google Style of Numpy Style docstring formats. 24 | 25 | ## Packaging 26 | 27 | M: Use a limited, unique, and well-defined symbol, macro, library, include, and/or module namespace. 28 | 29 | R: Package and provide automated builds using Spack and Conda 30 | 31 | ## Software design 32 | M: Provide configurable logging that adheres to standard logging approaches 33 | 34 | M: Use MPI in a way that is compatible with other products. 35 | 36 | M: Provide a runtime API to return the current version of the software and system configuration 37 | 38 | ## Documentation 39 | M: Include user documentation with the source code 40 | 41 | M: Include up-to-date user documentation in the release cycle of the documented source code 42 | 43 | M: User documentation should at least provide: (1) concise description of the project; (2) tutorial on how to use the documented code; and (3) details about how to contact the development team. 44 | 45 | M: Include a policy for handling pull requests from external contributors. 46 | 47 | R: Write documentation in a format that can be automatically converted into other formats. E.g., markdown or reStructuredText. 48 | 49 | R: Publish documentation in a web-based format, e.g., using readthedocs. 50 | 51 | ## Testing and continuous integration 52 | M: Provide a comprehensive test suite for verifying correctness of build and installation. 53 | 54 | M: Use regression tests in the development process. 55 | 56 | M: Use continuous integration (CI). 57 | 58 | R: Measure and record test coverage as part of CI 59 | 60 | ## Portability 61 | M: Give best effort at portability to common HPC platforms, schedulers, and software. 62 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | 2 | = ExaWorks: Software Development Kit 3 | 4 | An Exascale Computing Project team. 5 | 6 | == Overview 7 | 8 | The goal of this repo is to develop a Software Development Kit (SDK) for workflows at exascale. This will integrate the Parsl, Flux, RADICAL-Pilot, and Swift/T workflow systems, plus other tools, into a consistent set of "ExaWorks SDK Component Packages". Applications teams will be able to easily draw from these packages as needed, and develop and share workflows built on the various systems. 9 | 10 | == The SDK Workflow Systems in Brief 11 | 12 | === https://parsl-project.org[Parsl] 13 | 14 | Pure Python library for describing and managing parallel computation on one to many nodes. It contains abstractions to support various parallel and distributed workloads: from bag-of-tasks to dataflow, short to long duration tasks, single core through to multi-node. 15 | 16 | === http://flux-framework.org[Flux] 17 | 18 | Flux is a workload management system (à la Slurm, PBS, LSF), with 19 | single-user and multi-user (a.k.a. system instance) modes. 20 | 21 | === https://radical-cybertools.github.io[RADICAL-Cybertools] 22 | 23 | RADICAL-Cybertools is an abstractions-based suite of middleware building blocks that are architected for scalable, interoperable and sustainable approaches to support science on a range of high-performance and distributed computing systems. 24 | 25 | === http://swift-lang.org/Swift-T[Swift/T] 26 | 27 | Swift/T is an MPI-based workflow language and runtime system. It runs in a one big job model, with internal automatic task parallelization and data movement, enhanced by workflow-level compiler optimizations. 28 | 29 | == Objectives 30 | 31 | The objectives of this SDK are to: 32 | 33 | * Make it easy to install any or all of the workflow systems on the same underlying system. 34 | * Develop various levels of interoperabilty among the workflow systems. 35 | * Support Enterprise and HPC systems. 36 | 37 | == SDK Features 38 | 39 | === Dockerfiles 40 | 41 | Each workflow system has a Dockerfile in this repo to allow it to be easily deployed. 42 | 43 | === Continuous Integration / Continuous Deployment (CI/CD) 44 | 45 | The SDK maintains a CI/CD system for its supported component packages. The CI/CD tools will be portable to a variety of CI/CD services. For now, we use GitHub Actions CI workflows. 46 | 47 | === Packaging for HPC 48 | 49 | The SDK will support HPC-compatible distributions of its component packages via Spack. 50 | 51 | == Implementation 52 | 53 | === Dockerfiles 54 | 55 | ==== Usage of Base 56 | 57 | Each Dockerfile image relies on several packages from the ExaWorks SDK Base (CentOS 7), particularly the C tools and OpenMPI. Other Base images will be supported, including those based on CentOS 8. 58 | 59 | ==== Dependencies 60 | 61 | Most workflow system dependencies are in the Base Dockerfile, but some customizations may be made by the individual systems. 62 | 63 | ==== Build 64 | 65 | The SDK Base images can be built with a simple `docker build` command, using the directory `base/` as the build context. 66 | 67 | First, build the Base image (centos7|centos8) with: 68 | 69 | ---- 70 | $ cd docker/base/ 71 | $ docker build --tag --file centos7/Dockerfile . 72 | ---- 73 | 74 | Then, build the desired workflow system FROM the desired TAG. For example: 75 | 76 | ---- 77 | $ cd ../flux/ 78 | $ docker build --tag . 79 | ---- 80 | 81 | ==== Tests 82 | 83 | Each workflow system provides a small number of tests to verify that the build was successful. 84 | 85 | == Further Information 86 | 87 | Other ExaWorks resources including our public Slack channels are available through the https://exaworks.org[ExaWorks Home Page]. 88 | 89 | == Support 90 | 91 | You are welcome to provide feedback via the https://github.com/ExaWorks/SDK/discussions[ExaWorks SDK GitHub Discussion]. 92 | 93 | Please report technical issues to the https://github.com/ExaWorks/SDK/issues[ExaWorks SDK GitHub Issue Tracker]. 94 | -------------------------------------------------------------------------------- /ci/config/llnl/compilers.yaml: -------------------------------------------------------------------------------- 1 | compilers: 2 | - compiler: 3 | spec: gcc@9.4.0 4 | paths: 5 | cc: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/gcc-9.4.0-l7rjqbxn2lj54efqn22pizvhqbq433ay/bin/gcc 6 | cxx: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/gcc-9.4.0-l7rjqbxn2lj54efqn22pizvhqbq433ay/bin/g++ 7 | f77: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/gcc-9.4.0-l7rjqbxn2lj54efqn22pizvhqbq433ay/bin/gfortran 8 | fc: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/gcc-9.4.0-l7rjqbxn2lj54efqn22pizvhqbq433ay/bin/gfortran 9 | flags: {} 10 | operating_system: rhel7 11 | target: ppc64le 12 | modules: [] 13 | environment: {} 14 | extra_rpaths: [] 15 | - compiler: 16 | spec: gcc@8.1.0 17 | paths: 18 | cc: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.1.0-7p5pbywpk3zpsdad3tlvlshw6bqe67fp/bin/gcc 19 | cxx: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.1.0-7p5pbywpk3zpsdad3tlvlshw6bqe67fp/bin/g++ 20 | f77: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.1.0-7p5pbywpk3zpsdad3tlvlshw6bqe67fp/bin/gfortran 21 | fc: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.1.0-7p5pbywpk3zpsdad3tlvlshw6bqe67fp/bin/gfortran 22 | flags: {} 23 | operating_system: rhel7 24 | target: x86_64 25 | modules: [] 26 | environment: {} 27 | extra_rpaths: [] 28 | - compiler: 29 | spec: gcc@8.2.0 30 | paths: 31 | cc: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.2.0-dzdneu7z2l3lqtocwe76rxb62aeg3fuv/bin/gcc 32 | cxx: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.2.0-dzdneu7z2l3lqtocwe76rxb62aeg3fuv/bin/g++ 33 | f77: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.2.0-dzdneu7z2l3lqtocwe76rxb62aeg3fuv/bin/gfortran 34 | fc: /usr/WS1/exaworks/sdk/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/gcc-8.2.0-dzdneu7z2l3lqtocwe76rxb62aeg3fuv/bin/gfortran 35 | flags: {} 36 | operating_system: rhel7 37 | target: x86_64 38 | modules: [] 39 | environment: {} 40 | extra_rpaths: [] 41 | -------------------------------------------------------------------------------- /ci/config/llnl/config.yaml: -------------------------------------------------------------------------------- 1 | config: 2 | template_dirs: 3 | - $spack/share/spack/templates 4 | 5 | # Temporary locations Spack can try to use for builds. 6 | # 7 | # Recommended options are given below. 8 | # 9 | # Builds can be faster in temporary directories on some (e.g., HPC) systems. 10 | # Specifying `$tempdir` will ensure use of the default temporary directory 11 | # (i.e., ``$TMP` or ``$TMPDIR``). 12 | # 13 | # Another option that prevents conflicts and potential permission issues is 14 | # to specify `$user_cache_path/stage`, which ensures each user builds in their 15 | # home directory. 16 | # 17 | # A more traditional path uses the value of `$spack/var/spack/stage`, which 18 | # builds directly inside Spack's instance without staging them in a 19 | # temporary space. Problems with specifying a path inside a Spack instance 20 | # are that it precludes its use as a system package and its ability to be 21 | # pip installable. 22 | # 23 | # In any case, if the username is not already in the path, Spack will append 24 | # the value of `$user` in an attempt to avoid potential conflicts between 25 | # users in shared temporary spaces. 26 | # 27 | # The build stage can be purged with `spack clean --stage` and 28 | # `spack clean -a`, so it is important that the specified directory uniquely 29 | # identifies Spack staging to avoid accidentally wiping out non-Spack work. 30 | build_stage: 31 | - $tempdir/$user/spack-stage 32 | - $user_cache_path/stage 33 | # - $spack/var/spack/stage 34 | 35 | # Directory in which to run tests and store test results. 36 | # Tests will be stored in directories named by date/time and package 37 | # name/hash. 38 | test_stage: $user_cache_path/test 39 | source_cache: $spack/var/spack/cache 40 | misc_cache: $user_cache_path/cache 41 | connect_timeout: 10 42 | verify_ssl: true 43 | suppress_gpg_warnings: false 44 | install_missing_compilers: false 45 | checksum: true 46 | deprecated: false 47 | dirty: false 48 | build_language: C 49 | locks: true 50 | url_fetch_method: curl 51 | ccache: false 52 | concretizer: clingo 53 | db_lock_timeout: 3 54 | package_lock_timeout: null 55 | shared_linking: rpath 56 | allow_sgid: true 57 | terminal_title: false 58 | debug: false 59 | build_jobs: 16 60 | -------------------------------------------------------------------------------- /ci/config/llnl/packages.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | bzip2: 3 | externals: 4 | - spec: bzip2@1.0.6 5 | prefix: /usr 6 | gcc: 7 | externals: 8 | - spec: gcc@4.8.5 languages=c,c++,fortran 9 | prefix: /usr 10 | extra_attributes: 11 | compilers: 12 | c: /usr/bin/x86_64-redhat-linux-gcc 13 | cxx: /usr/bin/g++ 14 | fortran: /usr/bin/gfortran 15 | - spec: gcc@4.9.3 languages=c,c++,fortran 16 | prefix: /usr/tce 17 | extra_attributes: 18 | compilers: 19 | c: /usr/tce/bin/gcc 20 | cxx: /usr/tce/bin/g++ 21 | fortran: /usr/tce/bin/gfortran 22 | git: 23 | externals: 24 | - spec: git@1.8.3.1+tcltk 25 | prefix: /usr 26 | - spec: git@2.29.1+tcltk 27 | prefix: /usr/tce 28 | git-lfs: 29 | externals: 30 | - spec: git-lfs@2.12.0 31 | prefix: /usr/tce 32 | libtool: 33 | externals: 34 | - spec: libtool@2.4.2 35 | prefix: /usr 36 | m4: 37 | externals: 38 | - spec: m4@1.4.16 39 | prefix: /usr 40 | mvapich2: 41 | externals: 42 | - spec: mvapich2@2.3.1%intel@19.0.4.227~cuda~debug~regcache~wrapperrpath ch3_rank_bits=32 43 | fabrics=mrail file_systems=lustre,nfs,ufs process_managers=slurm 44 | prefix: /usr/tce/packages/mvapich2/mvapich2-2.3-intel-19.0.4 45 | openssh: 46 | externals: 47 | - spec: openssh@7.4p1 48 | prefix: /usr 49 | pkg-config: 50 | externals: 51 | - spec: pkg-config@0.27.1 52 | prefix: /usr 53 | tar: 54 | externals: 55 | - spec: tar@1.26 56 | prefix: /usr 57 | xz: 58 | externals: 59 | - spec: xz@5.2.2 60 | prefix: /usr 61 | -------------------------------------------------------------------------------- /ci/tests/flux/test.sh: -------------------------------------------------------------------------------- 1 | output=$(flux start flux mini run echo Success) 2 | if [[ "$output" != "Success" ]]; then 3 | exit 1 4 | fi 5 | 6 | echo "Success!" 7 | -------------------------------------------------------------------------------- /ci/tests/maestro/study.yaml: -------------------------------------------------------------------------------- 1 | description: # 2 | name: hello_bye_parameterized_funnel 3 | description: A study that says hello and bye to multiple people, and a final good bye to all. 4 | 5 | env: 6 | variables: # 7 | OUTPUT_PATH: /tmp/samples/hello_bye_parameterized_funnel 8 | labels: # 9 | HELLO_FORMAT: $(GREETING)_$(NAME).txt 10 | BYE_FORMAT: $(FAREWELL)_$(NAME).txt 11 | 12 | study: 13 | - name: say-hello 14 | description: Say hello to someone! 15 | run: # 16 | cmd: | 17 | echo "$(GREETING), $(NAME)!" > $(HELLO_FORMAT) 18 | 19 | - name: say-bye 20 | description: Say bye to someone! 21 | run: 22 | cmd: | 23 | echo "$(FAREWELL), $(NAME)!" > $(BYE_FORMAT) 24 | depends: [say-hello] # 25 | 26 | - name: bye-all 27 | description: Say bye to everyone! 28 | run: 29 | cmd: | 30 | echo "Good-bye, World!" > good_bye_all.txt 31 | depends: [say-bye_*] 32 | 33 | global.parameters: 34 | NAME: # 35 | values: [Pam, Jim, Michael, Dwight] 36 | label: NAME.%% 37 | GREETING: 38 | values: [Hello, Ciao, Hey, Hi] 39 | label: GREETING.%% 40 | FAREWELL: 41 | values: [Goodbye, Farewell, So long, See you later] 42 | label: FAREWELL.%% 43 | -------------------------------------------------------------------------------- /ci/tests/maestro/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # python3 -m virtualenv maestroenv 4 | 5 | # source maestroenv/bin/activate 6 | 7 | # pip install maestrowf 8 | 9 | 10 | > /tmp/test-output 11 | echo y | maestro run ci/tests/maestro/study.yaml >> /tmp/test-output 12 | 13 | cat </tmp/expected-output 14 | Would you like to launch the study? [yn] Study launched successfully. 15 | EOF 16 | 17 | if diff /tmp/test-output /tmp/expected-output; then 18 | echo "Output matches" 19 | else 20 | echo "Output does not match" 21 | echo "--- diff ---" 22 | diff /tmp/test-output /tmp/expected-output 23 | echo "------------" 24 | exit 1 25 | fi 26 | 27 | echo "Success!" 28 | -------------------------------------------------------------------------------- /ci/tests/parsl-flux/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | flux start python3 -m pytest $(python3 -c \ 6 | "import parsl.tests.test_flux as tf; print(tf.__file__)") \ 7 | --config=local --tap-stream 8 | -------------------------------------------------------------------------------- /ci/tests/parsl/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import parsl 4 | import os 5 | from parsl.app.app import python_app, bash_app 6 | from parsl.configs.local_threads import config 7 | 8 | parsl.load(config) 9 | 10 | @python_app 11 | def hello (): 12 | return 'Hello World from Python!' 13 | 14 | print(hello().result()) 15 | 16 | @bash_app 17 | def echo_hello(stdout='/tmp/echo-hello.stdout', stderr='/tmp/echo-hello.stderr'): 18 | return 'echo "Hello World!"' 19 | 20 | # avoid accumulation of results in persistent test environments 21 | try: 22 | os.unlink('/tmp/echo-hello.stdout') 23 | os.unlink('/tmp/echo-hello.stderr') 24 | except: 25 | pass 26 | 27 | echo_hello().result() 28 | 29 | with open('/tmp/echo-hello.stdout', 'r') as f: 30 | print(f.read()) 31 | -------------------------------------------------------------------------------- /ci/tests/parsl/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python ci/tests/parsl/test.py | tee /tmp/test-output 4 | 5 | cat </tmp/expected-output 6 | Hello World from Python! 7 | Hello World! 8 | 9 | EOF 10 | 11 | if diff /tmp/test-output /tmp/expected-output; then 12 | echo "Output matches" 13 | else 14 | echo "Output does not match" 15 | echo "--- diff ---" 16 | diff /tmp/test-output /tmp/expected-output 17 | echo "------------" 18 | exit 1 19 | fi 20 | 21 | echo "Success!" 22 | -------------------------------------------------------------------------------- /ci/tests/rp-flux/resource_flux.json: -------------------------------------------------------------------------------- 1 | { 2 | "localhost_flux": { 3 | "default_schema" : "local", 4 | "schemas" : { 5 | "local" : { 6 | "job_manager_endpoint": "fork://localhost/", 7 | "filesystem_endpoint" : "file://localhost/" 8 | } 9 | }, 10 | "resource_manager" : "FORK", 11 | "agent_scheduler" : "FLUX", 12 | "agent_spawner" : "FLUX", 13 | "launch_methods" : { 14 | "order" : ["FLUX", "FORK"], 15 | "FLUX" : {}, 16 | "FORK" : {} 17 | }, 18 | "default_remote_workdir" : "$HOME", 19 | "python_dist" : "default", 20 | "virtenv" : "$(pwd)/ve.rp", 21 | "virtenv_mode" : "create", 22 | "rp_version" : "local" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ci/tests/rp-flux/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if which radical-pilot-version >/dev/null; then 4 | rp_version="$(radical-pilot-version)" 5 | if [[ -z $rp_version ]]; then 6 | echo "RADICAL-Pilot version unknown" 7 | exit 1 8 | fi 9 | else 10 | echo "RADICAL-Pilot not installed" 11 | exit 1 12 | fi 13 | 14 | # each ci job runs in a private runner environment 15 | mkdir -p .radical/pilot/configs 16 | cp ci/tests/rp-flux/resource_flux.json .radical/pilot/configs/resource_flux.json 17 | export RADICAL_CONFIG_USER_DIR=$(pwd) 18 | export RADICAL_REPORT_ANIME=FALSE 19 | 20 | wget -q "https://raw.githubusercontent.com/radical-cybertools/radical.pilot/v$rp_version/examples/config.json" 21 | wget -q "https://raw.githubusercontent.com/radical-cybertools/radical.pilot/v$rp_version/examples/00_getting_started.py" 22 | chmod +x 00_getting_started.py 23 | 24 | radical-stack 25 | ./00_getting_started.py 'flux.localhost_flux' 26 | ret=$? 27 | echo "--- smoke test $ret" 28 | 29 | SID=$(ls -rt | grep rp.session | tail -1) 30 | test -z "$SID" || rm -rf "$HOME/radical.pilot.sandbox/$SID" 31 | echo '--- cleaned pilot sandbox' 32 | 33 | test "$ret" = 0 && echo "Success!" 34 | exit $ret 35 | 36 | -------------------------------------------------------------------------------- /ci/tests/rp-parsl/test.sh: -------------------------------------------------------------------------------- 1 | git clone -b feature/parsl_raptor_executor --single-branch https://github.com/radical-cybertools/radical.pilot.git 2 | 3 | mkdir -p ~/.radical/pilot/runs/ 4 | cp radical.pilot/examples/parsl/* ~/.radical/pilot/runs/ 5 | cp radical.pilot/examples/parsl/map_reduce.py ~/.radical/pilot/runs/ 6 | 7 | cd ~/.radical/pilot/runs/ 8 | 9 | [ -z ${RADICAL_PILOT_DBURL} ] && echo "\${RADICAL_PILOT_DBURL} not defined" 10 | 11 | echo '--- smoke test' 12 | chmod +x map_reduce.py 13 | python map_reduce.py 14 | ret=$? 15 | echo "--- smoke test $ret" 16 | 17 | test "$ret" = 0 && echo "Success!" 18 | exit $ret 19 | -------------------------------------------------------------------------------- /ci/tests/rp/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if which radical-pilot-version >/dev/null; then 4 | rp_version="$(radical-pilot-version)" 5 | if [[ -z $rp_version ]]; then 6 | echo "RADICAL-Pilot version unknown" 7 | exit 1 8 | fi 9 | else 10 | echo "RADICAL-Pilot not installed" 11 | exit 1 12 | fi 13 | 14 | # each ci job runs in a private runner environment 15 | mkdir -p .radical/pilot/configs 16 | cat > .radical/pilot/configs/resource_local.json < None: 45 | assert ismuex(start, end, command), 'Arguments are mutually exclusive' 46 | 47 | record = copy.deepcopy(self.record) 48 | 49 | if start: 50 | record['data'].update({ 51 | 'test_name': 'Set Environment', 52 | 'test_start_time': str(datetime.now()), 53 | 'test_end_time': str(datetime.now()), 54 | 'module': '_conftest', 55 | 'function': '_discover_environment', 56 | 'extras': { 57 | 'start_time': str(datetime.now()), # run_start_time 58 | 'git_branch': record['data']['branch'], 59 | 'config': { 60 | 'im_number': os.getenv('IM_NUMBER'), 61 | 'maintainer_email': os.getenv('MAINTAINER') 62 | } 63 | } 64 | }) 65 | 66 | elif end: 67 | record['data'].update({ 68 | 'test_name': 'End Test Series', 69 | 'test_start_time': str(datetime.now()), 70 | 'test_end_time': str(datetime.now()), 71 | 'module': '_conftest', 72 | 'function': '_end' 73 | }) 74 | 75 | elif command: 76 | tests_group = os.getenv('TESTS_GROUP', '').lower() 77 | name = kwargs.get('name') or command.split()[0] 78 | 79 | start_time = str(datetime.now()) 80 | results, out = self.execute_test(command) 81 | record['data'].update({ 82 | 'test_name': name, 83 | 'test_start_time': start_time, 84 | 'test_end_time': str(datetime.now()), 85 | 'module': tests_group, 86 | 'function': 'main', 87 | 'results': results, 88 | 'extras': { 89 | 'tests_group': tests_group 90 | } 91 | }) 92 | 93 | if kwargs.get('stdout'): 94 | print(out) 95 | 96 | print('### %s: %s' % (name, results['call']['status'])) 97 | 98 | else: 99 | raise RuntimeError('No viable option called, exiting...') 100 | 101 | self._session.post(self.dashboard_url, json=record, verify=False) 102 | 103 | @staticmethod 104 | def execute_test(command: str) -> Tuple[Dict, str]: 105 | results = {'setup': {'passed': True, 106 | 'status': 'passed', 107 | 'exception': None, 108 | 'report': ''}, 109 | 'call': {'passed': False, 110 | 'status': '', # passed, failed 111 | 'exception': None, 112 | 'report': ''}} 113 | 114 | try: 115 | out = subprocess.check_output(command, shell=True, 116 | stderr=subprocess.STDOUT, 117 | timeout=300) 118 | except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: 119 | out = e.output.decode('utf-8') if e.output else '' 120 | status = 'failed' 121 | exception = str(repr(e)) 122 | except KeyboardInterrupt as e: 123 | out = traceback.format_exc() 124 | status = 'failed' 125 | exception = str(repr(e)) 126 | else: 127 | out = out.decode('utf-8') if out else '' 128 | status = 'passed' 129 | exception = None 130 | 131 | passed = bool(status == 'passed') 132 | results['call'].update({'passed': passed, 133 | 'status': status, 134 | 'exception': exception, 135 | 'report': out if not passed else ''}) 136 | 137 | return results, out 138 | 139 | 140 | def ismuex(*a): 141 | return not bool(sum(map( 142 | lambda v: bool(v if isinstance(v, bool) else v is not None), a)) > 1) 143 | 144 | 145 | def get_args(): 146 | """ 147 | Get arguments. 148 | :return: Arguments namespace. 149 | :rtype: _AttributeHolder 150 | """ 151 | parser = argparse.ArgumentParser( 152 | description='Run SDK Tests by providing a corresponding command') 153 | 154 | test_group = parser.add_mutually_exclusive_group(required=True) 155 | test_group.add_argument( 156 | '-c', '--command', action='store', type=str, default=None, 157 | help='Command to be executed') 158 | test_group.add_argument( 159 | '-s', '--start', action='store_true', default=False, 160 | help='Start a series of test runs with the same id') 161 | test_group.add_argument( 162 | '-e', '--end', action='store_true', default=False, 163 | help='End a series of test runs with the same id') 164 | 165 | parser.add_argument( 166 | '-n', '--name', action='store', type=str, 167 | help='Name of the software tool (abbreviation)') 168 | parser.add_argument( 169 | '--stdout', action='store_true', default=False, 170 | help='Add STDOUT of the test run to the result') 171 | 172 | return parser.parse_args() 173 | 174 | 175 | class TransportAdapter(requests.adapters.HTTPAdapter): 176 | 177 | """ 178 | Transport adapter that allows to use custom ssl_context. 179 | """ 180 | 181 | def __init__(self, ssl_context=None, **kwargs): 182 | self.ssl_context = ssl_context 183 | super().__init__(**kwargs) 184 | 185 | def init_poolmanager(self, connections, maxsize, block=False, **kwargs): 186 | # save these values for pickling 187 | self._pool_connections = connections 188 | self._pool_maxsize = maxsize 189 | self._pool_block = block 190 | 191 | self.poolmanager = urllib3.poolmanager.PoolManager( 192 | num_pools=connections, maxsize=maxsize, block=block, 193 | ssl_context=self.ssl_context, **kwargs) 194 | 195 | 196 | if __name__ == '__main__': 197 | args = get_args() 198 | CITestsHandler().run(start=args.start, 199 | end=args.end, 200 | command=args.command, 201 | **{'name': args.name, 202 | 'stdout': args.stdout}) 203 | 204 | -------------------------------------------------------------------------------- /docker/base/centos7/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=centos:7 2 | FROM ${BASE_IMAGE} 3 | 4 | ARG MPI=openmpi 5 | ARG MPI_PREFIX=/usr/local 6 | ARG PYTHON_VERSION=3.7 7 | ARG PACKAGE_MANAGER=pip 8 | ENV VIRTUAL_ENV=/ve_exaworks 9 | ENV MPI_FLAVOR=${MPI} 10 | ENV MPI_PREFIX=${MPI_PREFIX} 11 | ENV MPICC=$MPI_PREFIX/bin/mpicc 12 | 13 | SHELL ["/bin/bash", "-l", "-c"] 14 | 15 | # General Dependencies 16 | RUN yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \ 17 | && yum -y update \ 18 | && yum -y install \ 19 | bzip2 \ 20 | coreutils \ 21 | curl \ 22 | file \ 23 | gcc \ 24 | gcc-c++ \ 25 | git \ 26 | man-db \ 27 | sudo \ 28 | vim \ 29 | wget \ 30 | which \ 31 | yum-utils 32 | 33 | RUN sed -i 's/http:\/\/vault.centos.org/https:\/\/vault.centos.org/g' /etc/yum.repos.d/* \ 34 | && yum upgrade ca-certificates 35 | 36 | # copy over the python installation script in case we need it for pip 37 | # conda will install its own python version 38 | COPY ./scripts/install-python.sh /scripts/ 39 | 40 | # The case of CentOS7 with `conda` package manager is excluded from CI workflow, 41 | # due to unresolved issues between GCC and OpenMPI installations 42 | RUN if [[ "${PACKAGE_MANAGER}" == "conda" ]] ; then \ 43 | yum -y install \ 44 | autoconf \ 45 | automake \ 46 | libtool \ 47 | make \ 48 | && yum clean all \ 49 | && V=8.5.0 \ 50 | && PKG=gcc-$V \ 51 | && URL=https://ftp.gnu.org/gnu/gcc \ 52 | && wget ${URL}/${PKG}/${PKG}.tar.gz \ 53 | && tar xf ${PKG}.tar.gz \ 54 | && cd ${PKG} \ 55 | && ./contrib/download_prerequisites \ 56 | && mkdir ../${PKG}-build \ 57 | && cd ../${PKG}-build \ 58 | && ../${PKG}/configure --disable-multilib \ 59 | --enable-languages=c,c++ \ 60 | && make -j 4 \ 61 | && make install \ 62 | && cd .. \ 63 | && rm -rf ${PKG}* \ 64 | && ldconfig ; \ 65 | else \ 66 | yum-builddep -y python3 \ 67 | && yum -y install \ 68 | # Flux-core Dependencies 69 | autoconf \ 70 | automake \ 71 | libtool \ 72 | make \ 73 | munge \ 74 | munge-devel \ 75 | cppcheck \ 76 | czmq-devel \ 77 | hwloc \ 78 | hwloc-devel \ 79 | jansson-devel \ 80 | sqlite-devel \ 81 | uuid-devel \ 82 | libuuid-devel \ 83 | libsodium-devel \ 84 | lua \ 85 | lua-devel \ 86 | lua-posix \ 87 | pkgconfig \ 88 | python3-pip \ 89 | sqlite \ 90 | lz4-devel \ 91 | jq \ 92 | # Flux-sched Dependencies 93 | boost-devel \ 94 | boost-graph \ 95 | boost-system \ 96 | boost-filesystem \ 97 | boost-regex \ 98 | libxml2-devel \ 99 | libedit-devel \ 100 | yaml-cpp-devel \ 101 | # Swift/T Dependencies 102 | java-1.7.0-openjdk-headless \ 103 | java-1.7.0-openjdk-devel \ 104 | tcl \ 105 | swig \ 106 | zsh \ 107 | && ./scripts/install-python.sh $PYTHON_VERSION \ 108 | && yum clean all ; \ 109 | fi 110 | 111 | COPY ./scripts/install-python-env.sh ./scripts/environment.yml /scripts/ 112 | RUN bash /scripts/install-python-env.sh ${PACKAGE_MANAGER} ${PYTHON_VERSION} 113 | 114 | # Swift/T Dependency on Apache Ant 115 | RUN wget https://archive.apache.org/dist/ant/binaries/apache-ant-1.9.15-bin.tar.gz \ 116 | && tar xvf apache-ant-1.9.15-bin.tar.gz -C /opt \ 117 | && ln -s /opt/apache-ant-1.9.15 /opt/ant \ 118 | && sudo ln -s /opt/ant/bin/ant /usr/bin/ant \ 119 | && rm apache-ant-1.9.15-bin.tar.gz 120 | 121 | COPY ./scripts/install-mpi.sh /scripts/install-mpi.sh 122 | RUN bash /scripts/install-mpi.sh ${MPI} 123 | -------------------------------------------------------------------------------- /docker/base/rockylinux8/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=rockylinux:8 2 | FROM ${BASE_IMAGE} 3 | 4 | ARG MPI=openmpi 5 | ARG MPI_PREFIX=/usr/local 6 | ARG PACKAGE_MANAGER=pip 7 | ARG PYTHON_VERSION=3.7 8 | ENV VIRTUAL_ENV=/ve_exaworks 9 | ENV MPI_FLAVOR=${MPI} 10 | ENV MPI_PREFIX=${MPI_PREFIX} 11 | ENV MPICC=$MPI_PREFIX/bin/mpicc 12 | 13 | SHELL ["/bin/bash", "-l", "-c"] 14 | 15 | # General Dependencies 16 | RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ 17 | && dnf install -y dnf-plugins-core \ 18 | && dnf config-manager --set-enabled powertools \ 19 | && dnf install -y \ 20 | bzip2 \ 21 | curl \ 22 | diffutils \ 23 | dnf-plugins-core \ 24 | file \ 25 | findutils \ 26 | gcc \ 27 | gcc-c++ \ 28 | git \ 29 | man-db \ 30 | sudo \ 31 | vim \ 32 | wget \ 33 | which 34 | 35 | # copy over the python installation script in case we need it for pip 36 | # conda will install its own python version 37 | COPY ./scripts/install-python.sh /scripts/ 38 | 39 | RUN if [[ "${PACKAGE_MANAGER}" == "conda" ]] ; then \ 40 | dnf clean all ; \ 41 | else \ 42 | dnf builddep -y python3 \ 43 | && dnf install -y \ 44 | # Flux-core Dependencies 45 | autoconf \ 46 | automake \ 47 | libtool \ 48 | make \ 49 | munge \ 50 | munge-devel \ 51 | cppcheck \ 52 | czmq-devel \ 53 | hwloc \ 54 | hwloc-devel \ 55 | jansson-devel \ 56 | sqlite-devel \ 57 | uuid-devel \ 58 | libuuid-devel \ 59 | libsodium-devel \ 60 | lua \ 61 | lua-devel \ 62 | lua-posix \ 63 | pkgconfig \ 64 | python3-pip \ 65 | sqlite \ 66 | lz4-devel \ 67 | jq \ 68 | # Flux-sched Dependencies 69 | boost-devel \ 70 | boost-graph \ 71 | boost-system \ 72 | boost-filesystem \ 73 | boost-regex \ 74 | libxml2-devel \ 75 | libedit-devel \ 76 | yaml-cpp-devel \ 77 | python3-pyyaml \ 78 | # Swift/T Dependencies 79 | java-1.8.0-openjdk-headless \ 80 | java-1.8.0-openjdk-devel \ 81 | tcl-devel \ 82 | swig \ 83 | zsh \ 84 | && ./scripts/install-python.sh $PYTHON_VERSION \ 85 | && dnf clean all ; \ 86 | fi 87 | 88 | COPY ./scripts/install-python-env.sh ./scripts/environment.yml /scripts/ 89 | RUN bash /scripts/install-python-env.sh ${PACKAGE_MANAGER} ${PYTHON_VERSION} 90 | 91 | # Swift/T Dependency on Apache Ant 92 | RUN wget https://archive.apache.org/dist/ant/binaries/apache-ant-1.9.15-bin.tar.gz \ 93 | && tar xvf apache-ant-1.9.15-bin.tar.gz -C /opt \ 94 | && ln -s /opt/apache-ant-1.9.15 /opt/ant \ 95 | && sudo ln -s /opt/ant/bin/ant /usr/bin/ant \ 96 | && rm apache-ant-1.9.15-bin.tar.gz 97 | 98 | COPY ./scripts/install-mpi.sh /scripts/install-mpi.sh 99 | RUN bash /scripts/install-mpi.sh ${MPI} 100 | -------------------------------------------------------------------------------- /docker/base/scripts/environment.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - anaconda 3 | - conda-forge 4 | dependencies: 5 | - autoconf 6 | - automake 7 | - boost 8 | - boost-cpp 9 | - "cffi>=1.1" 10 | - cryptography=3.3.2 11 | - "czmq>=3.0.1" 12 | - jansson=2.11 13 | - jsonschema=2 14 | - jq 15 | - libedit 16 | - libhwloc=1.11 17 | - libtool 18 | - libuuid 19 | - libxml2 20 | - lua=5.4 21 | - lua-luaposix 22 | - lz4 23 | - make 24 | - mpi 25 | - munge 26 | - openjdk=8 27 | - pip 28 | - pkgconfig 29 | - pytest 30 | - python=PY_VERSION 31 | - python_abi 32 | - "pyyaml>=3.10.0" 33 | - setuptools 34 | - six 35 | - "sqlite>=3.0.0" 36 | - swig 37 | - tk=8.6 38 | - valgrind 39 | - yaml-cpp 40 | - yaml-cpp-static 41 | - "zeromq>=4.0.4" 42 | - zlib 43 | - zsh 44 | - zstd 45 | 46 | # - mpich 47 | # - openmpi=4.0 48 | -------------------------------------------------------------------------------- /docker/base/scripts/install-mpi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z $1 ]]; then 4 | echo "Must provide MPI that you want to install" 1>&2 5 | exit 1 6 | fi 7 | 8 | # Get distribution info 9 | if [[ -f "/etc/centos-release" ]]; then 10 | DISTRO_ID="centos" 11 | DISTRO_MAJOR_VERSION=$(cat /etc/centos-release | cut -f 4 -d " " | cut -f 1 -d .) 12 | elif [[ -f "/etc/lsb-release" ]]; then 13 | DISTRO_ID=$(cat /etc/lsb-release | grep DISTRIB_ID | cut -f 2 -d "=" | awk '{print tolower($0)}') 14 | DISTRO_MAJOR_VERSION=$(cat /etc/lsb-release | grep DISTRIB_RELEASE | cut -f 2 -d "=" | cut -f 1 -d .) 15 | else 16 | echo "Unknown Linux distro. Exiting" 1>&2 17 | exit 1 18 | fi 19 | 20 | # Install only the dependencies for a given package 21 | # Source: https://serverfault.com/questions/429123/howto-get-yum-to-install-only-dependencies-for-a-given-pakage 22 | yum_install_only_deps () { 23 | if [[ -z "$1" ]]; then 24 | echo "Package required for installing deps" 1>&2 25 | exit 1 26 | fi 27 | PACKAGE="$1" 28 | yum deplist $PACKAGE | grep provider | awk '{print $2}' | sort | uniq | grep -v $PACKAGE | sed ':a;N;$!ba;s/\n/ /g' | xargs yum -y install 29 | } 30 | 31 | if [[ "$1" == "openmpi" ]]; then 32 | if [[ ${DISTRO_ID} == "centos" ]]; then 33 | if [[ ${DISTRO_MAJOR_VERSION} == "7" ]]; then 34 | yum install -y slurm-pmi-devel 35 | MAJOR_MINOR=1.10 36 | PATCH=7 37 | CONFIGURE_ARGS="--with-pmi --with-pmi-libdir=/usr/lib64" 38 | elif [[ ${DISTRO_MAJOR_VERSION} == "8" ]]; then 39 | yum_install_only_deps "openmpi" 40 | MAJOR_MINOR=4.0 41 | PATCH=6 42 | CONFIGURE_ARGS="" 43 | else 44 | echo "Unknown CentOS version. Exiting" 1>&2 45 | exit 1 46 | fi 47 | elif [[ ${DISTRO_ID} == "ubuntu" ]]; then 48 | if [[ ${DISTRO_MAJOR_VERSION} == "20" ]]; then 49 | apt-get update -y && apt install -y openmpi-bin 50 | MAJOR_MINOR=4.0 51 | PATCH=6 52 | CONFIGURE_ARGS="" 53 | else 54 | echo "Unknown Ubuntu version. Exiting" 1>&2 55 | exit 1 56 | fi 57 | fi 58 | 59 | OPENMPI=openmpi-${MAJOR_MINOR}.${PATCH} 60 | 61 | wget https://download.open-mpi.org/release/open-mpi/v${MAJOR_MINOR}/${OPENMPI}.tar.gz 62 | tar -xf ./${OPENMPI}.tar.gz 63 | rm ./${OPENMPI}.tar.gz 64 | 65 | cd ${OPENMPI} 66 | ./configure ${CONFIGURE_ARGS} 67 | make -j 4 68 | make install 69 | cd .. 70 | rm -rf ${OPENMPI} 71 | 72 | if [[ ${DISTRO_ID} == "centos" && ${DISTRO_MAJOR_VERSION} == "7" ]]; then 73 | yum remove -y slurm-pmi-devel 74 | yum autoremove -y 75 | yum clean all 76 | fi 77 | elif [[ "$1" == "mpich" ]]; then 78 | if [[ ${DISTRO_ID} == "centos" ]]; then 79 | yum install -y mpich-devel 80 | elif [[ ${DISTRO_ID} == "ubuntu" ]]; then 81 | apt-get update -y && apt install -y mpich dpkg-dev 82 | fi 83 | else 84 | printf "Unknown/unsupported MPI '%s'. Exiting without installing.\n" "$1" 85 | exit 1 86 | fi 87 | -------------------------------------------------------------------------------- /docker/base/scripts/install-python-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [[ -z $1 ]]; then 6 | echo "Must provide Package Manager that you want to install" 1>&2 7 | exit 1 8 | fi 9 | 10 | if [[ -z $2 ]]; then 11 | PY_BASE_VER=3.7 12 | else 13 | PY_BASE_VER=$2 14 | fi 15 | 16 | case "$PY_BASE_VER" in 17 | "3.7") 18 | PY_VER="3.7.12" 19 | ;; 20 | "3.8") 21 | PY_VER="3.8.12" 22 | ;; 23 | "3.9") 24 | PY_VER="3.9.7" 25 | ;; 26 | *) 27 | echo "Provided version is not supported" 1>&2 28 | exit 1 29 | esac 30 | 31 | if [[ "$1" == "conda" ]]; then 32 | curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh 33 | bash /tmp/miniconda.sh -bfp /usr/local && rm -rf /tmp/miniconda.sh 34 | source /usr/local/etc/profile.d/conda.sh 35 | conda update -y -n base conda 36 | sed -i "s/PY_VERSION/$PY_VER/" /scripts/environment.yml 37 | conda env create -p ${VIRTUAL_ENV} --file /scripts/environment.yml 38 | conda activate ${VIRTUAL_ENV} 39 | conda env config vars set BOOST_LIBDIR="$VIRTUAL_ENV/lib" \ 40 | LUA_INCLUDE="-I$VIRTUAL_ENV/include" \ 41 | LUA_LIB="$(pkg-config --libs lua)" \ 42 | C_INCLUDE_PATH="$VIRTUAL_ENV/include" \ 43 | CPLUS_INCLUDE_PATH="$VIRTUAL_ENV/include" \ 44 | LDFLAGS="-Wl,-rpath,$VIRTUAL_ENV/lib -L$VIRTUAL_ENV/lib" 45 | # finalize 46 | conda clean -y -a 47 | conda init --system 48 | echo "conda activate $VIRTUAL_ENV" >> ~/.bashrc 49 | echo "conda activate $VIRTUAL_ENV" >> /etc/profile.d/conda.sh 50 | 51 | elif [[ "$1" == "pip" ]]; then 52 | python3 -m venv ${VIRTUAL_ENV} 53 | source $VIRTUAL_ENV/bin/activate 54 | pip install --upgrade pip setuptools pytest 55 | # Flux python deps 56 | pip install cffi jsonschema pyyaml 57 | # Parsl python deps 58 | pip install "cryptography==3.3.2" 59 | echo "source $VIRTUAL_ENV/bin/activate" >> ~/.bashrc 60 | echo "source $VIRTUAL_ENV/bin/activate" >> /etc/profile.d/conda.sh 61 | 62 | else 63 | printf "Unknown/unsupported Python Package Manager '%s'. Exiting without installing.\n" "$1" 64 | exit 1 65 | fi 66 | -------------------------------------------------------------------------------- /docker/base/scripts/install-python.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | if [[ -z $1 ]]; then 6 | echo "Must provide python version (e.g., 3.6) to be installed" 1>&2 7 | exit 1 8 | fi 9 | 10 | # all dependencies should be resolved in base Dockerfile (including: openssl xz gdbm) 11 | # yum-based systems 12 | # yum -y install yum-utils 13 | # yum-builddep -y python3 14 | # dnf-based systems 15 | # dnf -y install dnf-plugins-core 16 | # dnf builddep -y python3 17 | # apt-based systems 18 | # apt-get -y install build-essential gdb lcov libbz2-dev libffi-dev libgdbm-dev \ 19 | # liblzma-dev libncurses5-dev libreadline6-dev libsqlite3-dev \ 20 | # libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g-dev 21 | 22 | PY_BASE_VER="$1" 23 | 24 | case "$PY_BASE_VER" in 25 | "3.7") 26 | PY_VER="3.7.12" 27 | ;; 28 | "3.8") 29 | PY_VER="3.8.12" 30 | ;; 31 | "3.9") 32 | PY_VER="3.9.7" 33 | ;; 34 | *) 35 | echo "Provided version is not supported" 1>&2 36 | exit 1 37 | esac 38 | 39 | PY_NAME="Python-$PY_VER" 40 | PY_TARBALL="$PY_NAME.tgz" 41 | PY_PREFIX="/usr" 42 | 43 | cd /tmp 44 | wget -q --no-check-certificate "https://www.python.org/ftp/python/$PY_VER/$PY_TARBALL" 45 | tar xzf "$PY_TARBALL" 46 | cd "/tmp/$PY_NAME" 47 | ./configure --enable-shared && make && make install 48 | ldconfig /usr/local/lib 49 | echo "ldconfig /usr/local/lib" >> ~/.bashrc 50 | 51 | if [[ ! "$PY_PREFIX" == "/usr" ]]; then 52 | ln -sf "$PY_PREFIX/bin/python$PY_BASE_VER" "/usr/bin/python3" 53 | ln -sf "$PY_PREFIX/bin/python$PY_BASE_VER" "/usr/bin/python$PY_BASE_VER" 54 | fi 55 | 56 | cd / && rm "/tmp/$PY_TARBALL" && rm -r "/tmp/$PY_NAME" 57 | -------------------------------------------------------------------------------- /docker/base/ubuntu2004/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=ubuntu:20.04 2 | FROM ${BASE_IMAGE} 3 | 4 | ARG MPI=openmpi 5 | ARG MPI_PREFIX=/usr/local 6 | ARG PACKAGE_MANAGER=pip 7 | ARG PYTHON_VERSION=3.7 8 | ENV VIRTUAL_ENV=/ve_exaworks 9 | ENV MPI_FLAVOR=${MPI} 10 | ENV MPI_PREFIX=${MPI_PREFIX} 11 | ENV MPICC=$MPI_PREFIX/bin/mpicc 12 | 13 | SHELL ["/bin/bash", "-l", "-c"] 14 | 15 | # General Dependencies 16 | RUN apt-get update -y \ 17 | && apt install -y \ 18 | bzip2 \ 19 | curl \ 20 | file \ 21 | gcc \ 22 | g++ \ 23 | git \ 24 | gnupg \ 25 | man-db \ 26 | sudo \ 27 | vim \ 28 | wget 29 | 30 | 31 | # copy over the python installation script in case we need it for pip 32 | # conda will install its own python version 33 | COPY ./scripts/install-python.sh /scripts/ 34 | 35 | RUN if [[ "${PACKAGE_MANAGER}" == "conda" ]] ; then \ 36 | apt-get update -y \ 37 | && apt install -y \ 38 | aspell \ 39 | aspell-en \ 40 | && apt-get clean ; \ 41 | else \ 42 | # install python 43 | ./scripts/install-python.sh $PYTHON_VERSION \ 44 | && apt-get update -y \ 45 | && DEBIAN_FRONTEND=noninteractive apt install -y \ 46 | # Flux-core Dependencies 47 | autoconf \ 48 | automake \ 49 | libtool \ 50 | make \ 51 | munge \ 52 | pkg-config \ 53 | libzmq3-dev \ 54 | libczmq-dev \ 55 | uuid-dev \ 56 | libjansson-dev \ 57 | liblz4-dev \ 58 | libhwloc-dev \ 59 | libsodium-dev \ 60 | libsqlite3-dev \ 61 | lua5.1 \ 62 | liblua5.1-dev \ 63 | lua-posix \ 64 | python3-cffi \ 65 | python3-yaml \ 66 | python3-jsonschema \ 67 | python3-sphinx \ 68 | python3-pip \ 69 | python3-venv \ 70 | aspell \ 71 | aspell-en \ 72 | valgrind \ 73 | libmpich-dev \ 74 | jq \ 75 | # Flux-sched Dependencies 76 | libboost-dev \ 77 | libboost-system-dev \ 78 | libboost-filesystem-dev \ 79 | libboost-graph-dev \ 80 | libboost-regex-dev \ 81 | libedit-dev \ 82 | libxml2-dev \ 83 | libyaml-cpp-dev \ 84 | # Swift/T Dependencies 85 | openjdk-8-jdk-headless \ 86 | tcl-dev \ 87 | swig \ 88 | zsh \ 89 | && apt-get clean ; \ 90 | fi 91 | 92 | COPY ./scripts/install-python-env.sh ./scripts/environment.yml /scripts/ 93 | RUN bash /scripts/install-python-env.sh ${PACKAGE_MANAGER} ${PYTHON_VERSION} 94 | 95 | # Swift/T Dependency on Apache Ant 96 | RUN wget https://archive.apache.org/dist/ant/binaries/apache-ant-1.9.15-bin.tar.gz \ 97 | && tar xvf apache-ant-1.9.15-bin.tar.gz -C /opt \ 98 | && ln -s /opt/apache-ant-1.9.15 /opt/ant \ 99 | && sudo ln -s /opt/ant/bin/ant /usr/bin/ant \ 100 | && rm apache-ant-1.9.15-bin.tar.gz 101 | 102 | COPY ./scripts/install-mpi.sh /scripts/install-mpi.sh 103 | RUN DEBIAN_FRONTEND=noninteractive bash /scripts/install-mpi.sh ${MPI} 104 | -------------------------------------------------------------------------------- /docker/flux/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=exaworks/sdk-base 2 | FROM ${BASE_IMAGE} 3 | 4 | ARG FLUX_CORE_VERSION=0.28.0 5 | ARG FLUX_SCHED_VERSION=0.17.0 6 | ENV FLUX_CORE_VERSION ${FLUX_CORE_VERSION} 7 | ENV FLUX_SCHED_VERSION ${FLUX_SCHED_VERSION} 8 | 9 | # Install Flux-core from release tarball 10 | RUN V=$FLUX_CORE_VERSION \ 11 | && PKG=flux-core-$V \ 12 | && URL=https://github.com/flux-framework/flux-core/releases/download \ 13 | && wget ${URL}/v${V}/${PKG}.tar.gz \ 14 | && tar xvf ${PKG}.tar.gz \ 15 | && cd ${PKG} \ 16 | && PYTHON_VERSION=3 ./configure --prefix=/usr \ 17 | && make -j 4 \ 18 | && make install \ 19 | && cd .. \ 20 | && rm -rf flux-core-* \ 21 | && ldconfig 22 | 23 | # Install Flux-sched from release tarball 24 | RUN BOOST_OPT=$([ -z "${BOOST_LIBDIR}" ] || echo "--with-boost-libdir=${BOOST_LIBDIR}") \ 25 | && V=$FLUX_SCHED_VERSION \ 26 | && PKG=flux-sched-$V \ 27 | && URL=https://github.com/flux-framework/flux-sched/releases/download \ 28 | && wget ${URL}/v${V}/${PKG}.tar.gz \ 29 | && tar xvf ${PKG}.tar.gz \ 30 | && cd ${PKG} \ 31 | && PYTHON_VERSION=3 ./configure ${BOOST_OPT} --prefix=/usr \ 32 | && make -j 4 \ 33 | && make install \ 34 | && cd .. \ 35 | && rm -rf flux-sched-* \ 36 | && ldconfig 37 | 38 | # Use the +/- features of env var expansion to only include ':$PYTHONPATH' if 39 | # $PYTHONPATH is set, otherwise it all expands to the empty string. 40 | # Note: we add both the `lib` and `lib64` directories to support centos7 and centos8, 41 | # respectively. For each distro, one of the directories is populated and the other 42 | # does not, but adding non-existent directories to PYTHONPATH has no correctness 43 | # implications, only (small) performance implications. 44 | ENV PYTHONPATH="/usr/lib64/flux/python3.6:/usr/lib/flux/python3.6${PYTHONPATH:+:}${PYTHONPATH:-}" 45 | 46 | COPY test.sh checks-annotate.sh /tests/flux/ 47 | CMD /tests/flux/test.sh 48 | -------------------------------------------------------------------------------- /docker/flux/checks-annotate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Post-process testsuite logs and outputs after a failure 4 | # 5 | # Uses GH Workflow commands for GH Actions 6 | # 7 | error() { 8 | printf "::error::$@\n" 9 | } 10 | catfile() { 11 | if test -f $1; then 12 | printf "::group::$1\n" 13 | cat $1 14 | printf "::endgroup::\n" 15 | fi 16 | } 17 | catfile_error() { 18 | error "Found $1" 19 | catfile $1 20 | } 21 | annotate_test_log() { 22 | # 23 | # Look through test logfiles for various failure indicators and 24 | # emit an annotation '::error::' to the logfile if found: 25 | # 26 | local test=$1 27 | 28 | # Emit an annotation for each failed test ('not ok') 29 | grep 'not ok' ${test}.log | while read line; do 30 | printf "::error file=${test}.t::%s\n" "${line}" 31 | done 32 | 33 | # Emit an annotation for TAP ERROR lines: 34 | grep '^ERROR: ' ${test}.log | while read line; do 35 | printf "::error file=${test}.t::%s\n" "${line}" 36 | done 37 | 38 | # Emit an annotation for chain-lint errors: 39 | grep '^error: bug in the test script' ${test}.log | while read line; do 40 | printf "::error file=${test}.t::%s\n" "${line}" 41 | done 42 | 43 | # Emit an annotation for anything that looks like an ASan error: 44 | sed -n 's/==[0-9][0-9]*==ERROR: //p' ${test}.log | while read line; do 45 | printf "::error file=${test}.t::%s\n" "${line}" 46 | done 47 | } 48 | 49 | # 50 | # Check all testsuite *.trs files and check for results that 51 | # were not 'SKIP' or 'PASS': 52 | # 53 | logfile=/tmp/check-errors.$$ 54 | cat /dev/null >$logfile 55 | 56 | errors=0 57 | total=0 58 | for trs in $(find . -name *.trs); do 59 | : $((total++)) 60 | result=$(sed -n 's/^.*global-test-result: *//p' ${trs}) 61 | if test "$result" != "PASS" -a "$result" != "SKIP"; then 62 | testbase=${trs//.trs} 63 | annotate_test_log $testbase >> $logfile 64 | catfile ${testbase}.output >> $logfile 65 | catfile ${testbase}.log >> $logfile 66 | : $((errors++)) 67 | fi 68 | done 69 | if test $errors -gt 0; then 70 | printf "::warning::" 71 | fi 72 | printf "Found ${errors} errors from ${total} tests in testsuite\n" 73 | cat $logfile 74 | rm $logfile 75 | 76 | # 77 | # Find and emit all *.asan.* files from test: 78 | # 79 | export -f catfile_error 80 | export -f catfile 81 | export -f error 82 | find . -name *.asan.* | xargs -i bash -c 'catfile_error {}' 83 | 84 | 85 | # 86 | # Check for any expected tests that were not run: 87 | # 88 | ls -1 ./*.t | sort >/tmp/expected 89 | ls -1 ./*.trs | sed 's/rs$//' | sort >/tmp/actual 90 | comm -23 /tmp/expected /tmp/actual > missing 91 | if test -s missing; then 92 | error "Detected $(wc -l missing) missing tests:" 93 | for f in $(cat missing); do 94 | printf "$f\n" 95 | file=${f//.t} 96 | test -f ${file}.log && catfile ${file}.log 97 | test -f ${file}.output && catfile ${file}.output 98 | done 99 | else 100 | printf "No missing test runs detected\n" 101 | fi 102 | 103 | -------------------------------------------------------------------------------- /docker/flux/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | output=$(flux start flux mini run echo Success) 4 | if [[ "$output" != "Success" ]]; then 5 | exit 1 6 | fi 7 | 8 | # When running with `flux start --test-size`, as the flux integration tests are, 9 | # openmpi hangs. This is because when Flux is started with `--test-size`, the 10 | # job-shell setting `pmi.clique` defaults to singleton, which means Flux's PMI 11 | # tells OpenMPI that all ranks are on the same node, even when you use `flux mini 12 | # run -N2`. This causes OpenMPI to hang. To solve this, add a job-shell plugin 13 | # that forces Flux's PMI to "lie" to OpenMPI that each shell is running on a 14 | # different node. OpenMPI properly bootstraps under this config. 15 | FLUX_LUA_D_DIR="$(dirname $(flux start flux getattr conf.shell_initrc))/lua.d" 16 | SHELL_PLUGIN_PATH="$FLUX_LUA_D_DIR/pmi-clique.lua" 17 | cat << EOF > $SHELL_PLUGIN_PATH 18 | pmiopt = shell.options.pmi or {} 19 | pmiopt["clique"] = "pershell" 20 | shell.options.pmi = pmiopt 21 | EOF 22 | chmod +x $SHELL_PLUGIN_PATH 23 | 24 | cd /tmp/ 25 | git clone https://github.com/flux-framework/flux-core.git 26 | cd flux-core 27 | if [[ -n "$FLUX_CORE_VERSION" ]]; then 28 | git checkout tags/"v${FLUX_CORE_VERSION}" 29 | fi 30 | ./autogen.sh 31 | ./configure 32 | make -j 2 33 | make -C src/common/libtap check 34 | cd t 35 | export MPI_TESTS="t2610-job-shell-mpir.t t3000-mpi-basic.t t3001-mpi-personalities.t t3003-mpi-abort.t" 36 | FLUXION_QMANAGER_RC_NOOP=t FLUXION_RESOURCE_RC_NOOP=t FLUX_TEST_INSTALLED_PATH=/usr/bin FLUX_TEST_MPI=t make check TESTS="$MPI_TESTS" 37 | exit_code=$? 38 | 39 | if [[ $exit_code -gt 0 ]]; then 40 | bash /tests/flux/checks-annotate.sh 41 | fi 42 | 43 | rm $SHELL_PLUGIN_PATH 44 | 45 | echo "Exiting with $exit_code" 46 | exit $exit_code 47 | -------------------------------------------------------------------------------- /docker/integration/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=exaworks/sdk-base 2 | FROM ${BASE_IMAGE} 3 | 4 | # Parsl-Flux integration 5 | 6 | # install pytest-tap to allow Parsl's pytest scripts to output TAP 7 | RUN python3 -m pip install pytest-tap 8 | COPY parsl-flux.sh /tests/parsl-flux/test.sh 9 | 10 | # RP-Flux integration 11 | 12 | RUN mkdir -p /root/.radical/pilot/configs/ 13 | COPY resource_flux.json /root/.radical/pilot/configs/resource_flux.json 14 | COPY rp-flux.sh /tests/rp-flux/test.sh 15 | 16 | CMD bash 17 | -------------------------------------------------------------------------------- /docker/integration/parsl-flux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | flux start python3 -m pytest $(python3 -c \ 6 | "import parsl.tests.test_flux as tf; print(tf.__file__)") \ 7 | --config=local --tap-stream 8 | -------------------------------------------------------------------------------- /docker/integration/resource_flux.json: -------------------------------------------------------------------------------- 1 | { 2 | "localhost_flux": { 3 | "default_schema" : "local", 4 | "schemas" : { 5 | "local" : { 6 | "job_manager_endpoint": "fork://localhost/", 7 | "filesystem_endpoint" : "file://localhost/" 8 | } 9 | }, 10 | "resource_manager" : "FORK", 11 | "agent_scheduler" : "FLUX", 12 | "agent_spawner" : "FLUX", 13 | "launch_methods" : { 14 | "order" : ["FLUX", "FORK"], 15 | "FLUX" : {}, 16 | "FORK" : {} 17 | }, 18 | "virtenv_mode" : "local" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /docker/integration/rp-flux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | radical-stack 4 | 5 | if [[ ! -d "/radical.pilot" ]]; then 6 | git clone -b "v$(radical-pilot-version)" --single-branch \ 7 | https://github.com/radical-cybertools/radical.pilot.git 8 | fi 9 | cd /radical.pilot 10 | 11 | # ensure path to flux-package 12 | PY_VER=$(python3 -c "import sys; print (sys.version[:3])") 13 | if [[ ! "$PY_VER" == *"$PYTHONPATH"* ]]; then 14 | export PYTHONPATH="/usr/lib/flux/python$PY_VER${PYTHONPATH:+:}${PYTHONPATH:-}" 15 | fi 16 | 17 | echo "--- smoke test" 18 | ./examples/00_getting_started.py "flux.localhost_flux" 19 | ret=$? 20 | echo "--- smoke test $ret" 21 | 22 | test "$ret" = 0 && echo "Success!" 23 | 24 | exit $ret 25 | 26 | -------------------------------------------------------------------------------- /docker/parsl/Dockerfile: -------------------------------------------------------------------------------- 1 | # Parsl addition to ExaWorks SDK 2 | # See https://parsl.readthedocs.io/en/stable/quickstart.html 3 | 4 | ARG BASE_IMAGE=exaworks/sdk-base 5 | FROM ${BASE_IMAGE} 6 | 7 | # Base has Python 3.6 installed 8 | 9 | # Recent versions of cryptography require Rust (the language) to be 10 | # installed. However, the version that comes with Centos 7 is too old. 11 | # Installing Rust using the install script on rust-lang.org requires 12 | # interactivity. 13 | # cyptography 3.3.2 appears to be a recent version that does not use 14 | # the rust build, so we'll use that until a more sensible solution 15 | # is found. 16 | 17 | RUN python3 -m pip install git+https://github.com/Parsl/parsl.git@master 18 | 19 | COPY test.py test.sh /tests/parsl/ 20 | CMD /tests/parsl/test.sh 21 | -------------------------------------------------------------------------------- /docker/parsl/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import parsl 4 | import os 5 | from parsl.app.app import python_app, bash_app 6 | from parsl.configs.local_threads import config 7 | 8 | parsl.load(config) 9 | 10 | @python_app 11 | def hello (): 12 | return 'Hello World from Python!' 13 | 14 | print(hello().result()) 15 | 16 | @bash_app 17 | def echo_hello(stdout='/tmp/echo-hello.stdout', stderr='/tmp/echo-hello.stderr'): 18 | return 'echo "Hello World!"' 19 | 20 | echo_hello().result() 21 | 22 | with open('/tmp/echo-hello.stdout', 'r') as f: 23 | print(f.read()) 24 | -------------------------------------------------------------------------------- /docker/parsl/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python /tests/parsl/test.py | tee /tmp/test-output 4 | 5 | cat </tmp/expected-output 6 | Hello World from Python! 7 | Hello World! 8 | 9 | EOF 10 | 11 | if diff /tmp/test-output /tmp/expected-output; then 12 | echo "Output matches" 13 | else 14 | echo "Output does not match" 15 | exit 1 16 | fi 17 | -------------------------------------------------------------------------------- /docker/rct/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | # RCT DOCKERFILE 3 | 4 | ARG BASE_IMAGE=exaworks/sdk-base 5 | FROM ${BASE_IMAGE} 6 | 7 | # RCT installation (RADICAL-EnTK includes RADICAL-Pilot) 8 | RUN pip install \ 9 | radical.entk 10 | 11 | # clone RP and RE to get the test suite 12 | RUN git clone -b master --single-branch https://github.com/radical-cybertools/radical.pilot.git \ 13 | && git clone -b master --single-branch https://github.com/radical-cybertools/radical.entk.git 14 | 15 | COPY run_rp_test.sh /tests/rp/test.sh 16 | COPY run_re_test.sh /tests/re/test.sh 17 | 18 | # run RP tests 19 | CMD /bin/sh /tests/rp/test.sh 20 | -------------------------------------------------------------------------------- /docker/rct/run_re_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | radical-stack 4 | 5 | cd radical.entk 6 | echo '--- smoke test' 7 | ./examples/user_guide/get_started.py 8 | ret=$? 9 | echo "--- smoke test $ret" 10 | 11 | exit $ret 12 | 13 | -------------------------------------------------------------------------------- /docker/rct/run_rp_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | radical-stack 4 | 5 | cd /radical.pilot 6 | echo '--- smoke test' 7 | ./examples/00_getting_started.py 8 | ret=$? 9 | echo "--- smoke test $ret" 10 | 11 | exit $ret 12 | 13 | -------------------------------------------------------------------------------- /docker/swift-t/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | # SWIFT/T DOCKERFILE 3 | # Follows instructions at: 4 | # http://swift-lang.github.io/swift-t/guide.html#install_source 5 | 6 | ARG BASE_IMAGE=exaworks/sdk-base 7 | FROM ${BASE_IMAGE} 8 | 9 | # Customize the Swift/T root directory here: 10 | ENV SWIFT_ROOT=/opt/swift-t 11 | 12 | # Setup build environment 13 | ARG SWIFT_BUILD_DIR=/tmp/build-swift-t 14 | RUN mkdir -pv ${SWIFT_BUILD_DIR} 15 | WORKDIR ${SWIFT_BUILD_DIR} 16 | COPY install-deps.sh install-swift-t.sh ${SWIFT_BUILD_DIR}/ 17 | 18 | RUN ./install-deps.sh 19 | 20 | ENV PATH=$SWIFT_ROOT/stc/bin:$PATH 21 | 22 | RUN ./install-swift-t.sh 23 | WORKDIR / 24 | 25 | # Simple sanity tests 26 | COPY test-sanity.sh /tests/swift/test.sh 27 | CMD /tests/swift/test.sh 28 | -------------------------------------------------------------------------------- /docker/swift-t/README.adoc: -------------------------------------------------------------------------------- 1 | 2 | = Dockerfile: Swift/T 3 | 4 | == Overview 5 | 6 | This Dockerfile will install Swift/T using the OpenMPI provided by the ExaWorks SDK Base image. It is a vanilla installation with no integrated language support. 7 | 8 | == Usage 9 | 10 | ---- 11 | $ docker build . 12 | ---- 13 | 14 | == Method 15 | 16 | === Usage of Base 17 | 18 | This image relies on several packages from the ExaWorks SDK Base (Centos 7), particularly the C tools and OpenMPI. 19 | 20 | === Dependencies 21 | 22 | Dependencies are considered in the shell script `install-deps.sh`. 23 | 24 | ZSH and SWIG are installed from Yum. 25 | 26 | Swift/T relies on Tcl 8.6, which is not supported in Centos 7, so we build that manually. 27 | 28 | === Build 29 | 30 | Swift/T is built from source in `install-swift-t.sh`. 31 | 32 | This is a standard build from the Swift/T GitHub repo. `sed` is used to populate the Swift/T settings file before the build. 33 | 34 | You can find more information about compiling Swift/T http://swift-lang.github.io/swift-t/guide.html#install_source[here]. 35 | 36 | == Tests 37 | 38 | Currently, a small test suite is run from shell script `test-sanity.sh`. 39 | 40 | These tests: 41 | 42 | . tell Swift/T to show its build settings and 43 | . run a simple single-command workflow from a string. 44 | 45 | Because of the Swift/T architecture, this is enough to capture and diagnose most build problems, such as the use of inconsistent MPI or Java settings. If these work, it is almost certain that other language uses will work. 46 | 47 | == Further information 48 | 49 | * http://swift-lang.org/Swift-T[Swift/T Front Page] 50 | + 51 | Main page for Swift/T information. 52 | + 53 | * http://swift-lang.github.io/swift-t/guide.html[Swift/T Guide] 54 | + 55 | The detailed manual. 56 | + 57 | * http://swift-lang.github.io/swift-t/gallery.html[Swift/T Gallery] 58 | + 59 | Some example programs. 60 | + 61 | * https://github.com/swift-lang/swift-t[Swift/T GitHub] 62 | + 63 | The repository, issue tracker, etc. 64 | -------------------------------------------------------------------------------- /docker/swift-t/install-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eux 3 | 4 | # INSTALL DEPENDENCIES 5 | # Currently just Tcl 6 | 7 | if which tclsh8.6 > /dev/null 2>&1 8 | then 9 | # Tcl is installed, probably via package manager. 10 | # Note that this does not check for tcl-devel, 11 | # which is needed for Swift/T. 12 | exit 13 | fi 14 | 15 | # Get Tcl 16 | wget --no-verbose https://prdownloads.sourceforge.net/tcl/tcl8.6.11-src.tar.gz 17 | tar xfz tcl8.6.11-src.tar.gz 18 | 19 | # Fix Tcl RTLD setting: 20 | cd tcl8.6.11/unix 21 | sed -i 90s@^@//@ tclLoadDl.c 22 | sed -i 92,+2s@^@//@ tclLoadDl.c 23 | 24 | # Build Tcl 25 | ./configure --prefix=/opt/tcl-8.6.11 26 | make binaries libraries 27 | make install-binaries install-libraries install-headers 28 | -------------------------------------------------------------------------------- /docker/swift-t/install-swift-t.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eux 3 | 4 | # INSTALL SWIFT/T 5 | 6 | # Setup 7 | # Swift/T will prefer the MPI found first- MPICH in this case 8 | PATH=/usr/lib64/mpich/bin:$PATH 9 | PATH=/usr/lib64/openmpi/bin:$PATH 10 | PATH=/opt/tcl-8.6.11/bin:$PATH 11 | 12 | # Get it 13 | git clone https://github.com/swift-lang/swift-t.git 14 | 15 | # Build it 16 | cd swift-t/dev/build 17 | cp -v swift-t-settings.sh.template swift-t-settings.sh 18 | sed -i "/SWIFT_T_PREFIX=/s@=.*@=$SWIFT_ROOT@" swift-t-settings.sh 19 | ./build-swift-t.sh 20 | -------------------------------------------------------------------------------- /docker/swift-t/test-sanity.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | # SWIFT/T TEST SANITY 5 | # Simply sanity tests for fresh Docker build 6 | 7 | # For OpenMPI 8 | if [[ $MPI_FLAVOR == "openmpi" ]]; then 9 | export TURBINE_LAUNCH_OPTIONS=--allow-run-as-root 10 | fi 11 | 12 | swift-t -v 13 | swift-t -E 'trace(42);' 14 | -------------------------------------------------------------------------------- /docker/tutorials/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | # https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope 3 | ARG BUILDPLATFORM=linux/amd64 4 | ARG PY_VER=3.9 5 | 6 | FROM --platform=${BUILDPLATFORM} jupyter/minimal-notebook:python-${PY_VER} 7 | 8 | ENV JUPYTER_ENABLE_LAB=yes 9 | 10 | USER root 11 | 12 | RUN apt-get update -y \ 13 | && apt-get install -y bc curl gnupg 14 | 15 | # General jupyter deps 16 | 17 | RUN mamba install -y -n base -c conda-forge \ 18 | ipywidgets \ 19 | jupyterlab_widgets \ 20 | nb_conda_kernels \ 21 | nodejs \ 22 | && mamba clean -a -f -y \ 23 | && jupyter labextension install @jupyter-widgets/jupyterlab-manager \ 24 | && fix-permissions "${CONDA_DIR}" \ 25 | && fix-permissions "/home/${NB_USER}" 26 | 27 | USER ${NB_UID} 28 | 29 | # Config for nb_conda_kernels 30 | 31 | RUN printf '\n\ 32 | { \n\ 33 | "CondaKernelSpecManager": { \n\ 34 | "kernelspec_path": "--user" \n\ 35 | } \n\ 36 | } \n\ 37 | ' > $(jupyter --config-dir)/jupyter_config.json \ 38 | && jupyter nbextension install --user --py widgetsnbextension 39 | 40 | # Tutorials 41 | 42 | WORKDIR /tutorials/ 43 | COPY --chown=${NB_UID}:${NB_GID} ./ ./ 44 | 45 | # RADICAL-Cybertools (RADICAL-EnTK and RADICAL-Pilot) 46 | 47 | RUN mamba install -y -c conda-forge \ 48 | 'radical.entk>=1.40' 49 | 50 | # Parsl-RP integration usecase NWChem 51 | 52 | RUN mamba install -y -c conda-forge \ 53 | nwchem 54 | 55 | # PSI/J - MPI example 56 | 57 | RUN mamba install -y -c conda-forge \ 58 | gxx_linux-64 \ 59 | openmpi 60 | 61 | # Swift/T 62 | 63 | RUN mamba install -y -c conda-forge \ 64 | gcc zsh openjdk=11 openmpi numpy \ 65 | && mamba install -y -c conda-forge \ 66 | http://web.cels.anl.gov/~woz/swift-t-1.6.0-py39_1.tar.bz2 67 | 68 | # Clonning repos and creation of environments from there 69 | 70 | RUN git clone -b main --single-branch \ 71 | https://github.com/ExaWorks/Tutorial.git \ 72 | /tutorials/sdk-examples \ 73 | && git clone -b main --single-branch \ 74 | https://github.com/ExaWorks/molecular-design-parsl-demo \ 75 | /tutorials/molecular-design-parsl-demo 76 | 77 | # Parsl 78 | 79 | RUN cd /tutorials/molecular-design-parsl-demo \ 80 | && sed -i '1d' environment.yml \ 81 | && mamba env create -n parsl -f environment.yml \ 82 | && mamba update -y -n parsl \ 83 | jupyterlab_widgets \ 84 | ipywidgets 85 | 86 | -------------------------------------------------------------------------------- /docker/tutorials/README.md: -------------------------------------------------------------------------------- 1 | # Docker container with tutorial notebooks 2 | 3 | ## Build container image 4 | 5 | SDK Tutorials container is based on 6 | [jupyter/minimal-notebook](https://github.com/jupyter/docker-stacks) image. 7 | 8 | ```shell 9 | ./docker/tutorials/build.sh 10 | ``` 11 | 12 | **NOTE**: for ARM platform, please, pull the image from the DockerHub directly 13 | 14 | ```shell 15 | # use a specific tag if needed, otherwise "latest" is used (by default) 16 | docker pull exaworks/sdk-tutorials:latest 17 | ``` 18 | 19 | ## Run container image 20 | 21 | ```shell 22 | docker run --rm -it -p 8888:8888 exaworks/sdk-tutorials 23 | # OR 24 | # run.sh [tag_name] 25 | # OR with mounted tutorials directory 26 | # run-local.sh [tag_name] 27 | ``` 28 | 29 | -------------------------------------------------------------------------------- /docker/tutorials/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SDK_BASE_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )/../../" &> /dev/null && pwd 2> /dev/null; )" 4 | 5 | docker build \ 6 | -t exaworks/sdk-tutorials \ 7 | -f "$SDK_BASE_DIR/docker/tutorials/Dockerfile" \ 8 | "$SDK_BASE_DIR/docs/source/tutorials/" 9 | -------------------------------------------------------------------------------- /docker/tutorials/push-multi-platform.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SDK_BASE_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )/../../" &> /dev/null && pwd 2> /dev/null; )" 4 | 5 | TAG="exaworks/sdk-tutorials:" 6 | if [[ -z $1 ]]; then 7 | TAG+="${SDK_TUTORIALS_TAG:-latest}" 8 | else 9 | TAG+=$1 10 | fi 11 | 12 | echo "Push docker container $TAG" 13 | 14 | # push to the DockerHub registry 15 | docker buildx create --use --name sdk_builder 16 | docker buildx build \ 17 | --output=type=registry \ 18 | --platform linux/amd64,linux/arm64 \ 19 | -t "$TAG" \ 20 | -f "$SDK_BASE_DIR/docker/tutorials/Dockerfile" \ 21 | "$SDK_BASE_DIR/docs/source/tutorials/" 22 | -------------------------------------------------------------------------------- /docker/tutorials/run-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TAG="exaworks/sdk-tutorials:" 4 | if [[ -z $1 ]]; then 5 | TAG+="${SDK_TUTORIALS_TAG:-latest}" 6 | else 7 | TAG+=$1 8 | fi 9 | 10 | echo "Run docker container $TAG" 11 | 12 | # Runs docker but mounts the git /tutorials dir in /tutorials such that changes 13 | # in the notebook are saved 14 | docker run --rm -it -p 8888:8888 --mount type=bind,source="$(pwd)/../../docs/source/tutorials",target=/tutorials -w /tutorials "$TAG" 15 | -------------------------------------------------------------------------------- /docker/tutorials/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TAG="exaworks/sdk-tutorials:" 4 | if [[ -z $1 ]]; then 5 | TAG+="${SDK_TUTORIALS_TAG:-latest}" 6 | else 7 | TAG+=$1 8 | fi 9 | 10 | echo "Run docker container $TAG" 11 | 12 | docker run --rm -it -p 8888:8888 -w /tutorials "$TAG" 13 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.https://www.sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/source/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | .wy-side-nav-search { 2 | background-color: #de6262 !important; 3 | } 4 | 5 | .wy-side-nav-search>div.version { 6 | color: hsla(0,0%,100%,.5); 7 | } 8 | 9 | .caption-text { 10 | color: #F77F15 !important; 11 | } 12 | 13 | .wy-side-nav-search input[type=text] { 14 | border-color: #de6262; 15 | } 16 | 17 | /* .wy-nav-side { 18 | background-image: -moz-linear-gradient(180deg, #de6262 15%, #ffb88c 55%); 19 | background-image: -webkit-linear-gradient(180deg, #de6262 15%, #ffb88c 55%); 20 | background-image: -ms-linear-gradient(180deg, #de6262 15%, #ffb88c 55%); 21 | background-image: linear-gradient(180deg, #de6262 15%, #ffb88c 55%); 22 | background-color: #de6262; 23 | } */ 24 | 25 | .cards { 26 | display: flex; 27 | flex-direction: row; 28 | flex-wrap: wrap; 29 | } 30 | 31 | .card { 32 | max-width: 45%; 33 | margin-right: 1em; 34 | margin-bottom: 1em; 35 | padding: 1em; 36 | box-shadow: 0px 4px 8px rgba(0, 0, 0, 0.04), 0px 0px 2px rgba(0, 0, 0, 0.06), 0px 0px 1px rgba(0, 0, 0, 0.04) !important; 37 | border-radius: 0.5em; 38 | } 39 | 40 | .card h3 { 41 | font-size: 1.1em; 42 | font-weight: 500; 43 | text-align: center; 44 | margin-top: 1em; 45 | } 46 | 47 | .card p { 48 | padding: 0; 49 | margin: 0; 50 | font-size: 0.9em; 51 | } 52 | 53 | .icon-logo { 54 | font-size: 2em; 55 | text-align: center; 56 | width: 100%; 57 | } 58 | -------------------------------------------------------------------------------- /docs/source/components.rst: -------------------------------------------------------------------------------- 1 | .. _chapter_components: 2 | 3 | =============== 4 | Core Components 5 | =============== 6 | 7 | The SDK has four core components but it is open to the contribution of any system that support the execution of scientific workflows on the Department of Energy high performance computing platforms. 8 | 9 | * `Flux `_. Workload management system (à la Slurm, 10 | PBS, LSF), with single-user and multi-user (a.k.a. system instance) modes. 11 | * `Parsl `_. Pure Python library for describing and 12 | managing parallel computation on one to many nodes. It contains abstractions 13 | to support various parallel and distributed workloads: from bag-of-tasks to 14 | dataflow, short to long duration tasks, single core through to multi-node. 15 | * `PSI/J `_. The Portable Submission 16 | Interface for Jobs is a Python abstraction layer over cluster schedulers. A 17 | number of executors and launchers allow PSI/J to communicate with specific job 18 | schedulers. 19 | * `RADICAL-Cybertools `_. 20 | `RADICAL-EnsembleToolkit (EnTK) 21 | `_ and `RADICAL-Pilot (RP) 22 | `_ are middleware architected 23 | for scalability, interoperability and sustainability. Implemented as Python 24 | modules, they support the execution of scientific workflows and workloads on a 25 | range of high-performance and distributed computing platforms. 26 | * `Swift/T `_. Swift/T is an MPI-based workflow 27 | language and runtime system. It runs in a one big job model, with internal 28 | automatic task parallelization and data movement, enhanced by workflow-level 29 | compiler optimizations. 30 | 31 | Each core component can be indipendently installed by following the instructions 32 | of each component's documentation. Exaworks SDK curates the containarization of 33 | each component and its `Spack 34 | `_ packaging. 35 | 36 | Containers 37 | ---------- 38 | 39 | ExaWorks SDK packages are available via container from `Docker hub 40 | `_. The following code shows how to access 41 | the SDK container image locally without installation. 42 | 43 | :: 44 | 45 | docker pull exaworks/sdk 46 | docker run -it exaworks/sdk bash 47 | 48 | The following code shows how to use the container images to run the notebook 49 | tutorials. Note that the specific notebook you want to run may have some 50 | additional prerequisites. 51 | 52 | :: 53 | 54 | docker run -p 8888:8888 -v path/to/notebooks:/notebooks -it exaworks/sdk bash 55 | pip install jupyter 56 | cd /notebooks 57 | jupyter notebook --allow-root --ip 0.0.0.0 --no-browser 58 | 59 | Spack packages 60 | -------------- 61 | 62 | ExaWorks SDK packages are packed together into Spack ``exaworks`` package. The 63 | following code shows its installation within a corresponding Spack environment 64 | 65 | :: 66 | 67 | spack env create exaworkssdk 68 | spack env activate exaworkssdk 69 | spack install exaworks 70 | 71 | If Spack is not in the system, then it could be installed manually 72 | 73 | :: 74 | 75 | git clone https://github.com/spack/spack.git 76 | . spack/share/spack/setup-env.sh 77 | 78 | Steps for package creation are provided in `Packaging section `_. 79 | For additional information please refer to `the Spack documentation `_. -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | import sphinx_rtd_theme 17 | from sphinx.ext.apidoc import main 18 | 19 | needs_sphinx = '1.6' 20 | 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = 'ExaWorks SDK' 25 | copyright = '2022, ExaWorks Team' 26 | author = 'ExaWorks Team' 27 | 28 | # The full version, including alpha/beta/rc tags 29 | release = '1.0' 30 | version = release 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'sphinx.ext.duration', 39 | 'sphinx.ext.napoleon', 40 | 'sphinx.ext.autodoc', 41 | 'sphinx.ext.doctest', 42 | 'sphinx.ext.intersphinx', 43 | 'sphinx.ext.todo', 44 | 'sphinx.ext.coverage', 45 | 'sphinx.ext.mathjax', 46 | 'sphinx.ext.ifconfig', 47 | 'sphinx.ext.viewcode', 48 | 'sphinx.ext.githubpages', 49 | 'nbsphinx', 50 | 'sphinx_copybutton', 51 | 'sphinx_rtd_theme', 52 | 'sphinx.ext.autodoc.typehints', 53 | 'sphinx.ext.intersphinx', 54 | 'sphinx.ext.viewcode' 55 | ] 56 | 57 | # Add any paths that contain templates here, relative to this directory. 58 | templates_path = ['_templates'] 59 | 60 | # List of patterns, relative to source directory, that match files and 61 | # directories to ignore when looking for source files. 62 | # This pattern also affects html_static_path and html_extra_path. 63 | exclude_patterns = [] 64 | 65 | 66 | # -- Options for HTML output ------------------------------------------------- 67 | 68 | # The theme to use for HTML and HTML Help pages. See the documentation for 69 | # a list of builtin themes. 70 | # 71 | html_theme = "sphinx_rtd_theme" 72 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 73 | 74 | # Add any paths that contain custom static files (such as style sheets) here, 75 | # relative to this directory. They are copied after the builtin static files, 76 | # so a file named "default.css" will overwrite the builtin "default.css". 77 | html_static_path = ['_static'] 78 | 79 | html_favicon = 'favicon.ico' 80 | html_css_files = [ 81 | 'css/custom.css', 82 | ] 83 | # html_logo = "images/exaworks-psi-j-logo-white.png" 84 | html_theme_options = { 85 | 'logo_only': True, 86 | 'display_version': True, 87 | } 88 | autoclass_content = 'both' 89 | add_module_names = False 90 | python_use_unqualified_type_names = True 91 | autodoc_mock_imports = ['flux'] 92 | nitpick_ignore = [ 93 | ('py:class', 'distutils.version.StrictVersion'), 94 | ('py:class', 'distutils.version.Version') 95 | ] 96 | -------------------------------------------------------------------------------- /docs/source/contribute.rst: -------------------------------------------------------------------------------- 1 | .. _chapter_contributing: 2 | 3 | =================== 4 | Contributing to SDK 5 | =================== 6 | 7 | This document outlines the policies and recommendations for inclusion in the 8 | ExaWorks SDK. The policies are based on those defined by the xSDK and LLNL 9 | RADIUSS projects and reflect best practices for open source development, 10 | development of sustainable software, and scalable deployment of software on ECP 11 | systems. 12 | 13 | - **M** indicates a mandatory policy 14 | - **R** indicates a recommended policy 15 | 16 | Licensing 17 | --------- 18 | 19 | - M: Use an OSI-approved open-source license (e.g., Apache, MIT, BSD, LGPL) 20 | - R: Provide a list of dependencies and their licenses in a standard format 21 | (e.g., SPDX) 22 | 23 | Code 24 | ---- 25 | 26 | - M: Code should be version controlled and publicly accessible online 27 | - M: Provide a transparent, online contribution process based on published 28 | contributing guide, and using pull requests and issues collection 29 | - R: Code should be version controlled using Git and accessible on GitHub 30 | - R: Follow a common style guide for code layout, documentation, naming, etc. 31 | (e.g., PEP8) 32 | 33 | Packaging 34 | --------- 35 | 36 | - M: Package and provide automated builds using Spack and Conda 37 | - M: Use a limited, unique, and well-defined symbol, macro, library, include, 38 | and/or module namespace. 39 | 40 | Software design 41 | --------------- 42 | 43 | - M: Provide configurable logging that adheres to standard logging approaches. 44 | - M: Use MPI in a way that is compatible with other products, i.e., multiple 45 | tools using MPI at the same time vs. leveraging multiple MPI implementations). 46 | - M: Provide a runtime API to return the current version of the software and 47 | system configuration. 48 | 49 | Documentation 50 | ------------- 51 | 52 | - M: Publish documentation in a web-based format. 53 | - M: Provide a concise description of the project. 54 | - M: Version control documentation consistent with and alongside source code. 55 | - M: Provide a documented, reliable way to contact the development team. 56 | - M: Provide and maintain example source code along with documentation. 57 | - M: Provide a documented policy for handling pull requests from external 58 | contributors. 59 | 60 | Testing and continuous integration 61 | ---------------------------------- 62 | 63 | - M: Provide a comprehensive test suite for verifying correctness of build and 64 | installation. 65 | - M: Use regression tests in the development process. 66 | - M: Use continuous integration (CI). 67 | - R: Measure and record test coverage as part of CI 68 | 69 | Portability 70 | ----------- 71 | 72 | - M: Give best effort at portability to common HPC platforms, schedulers, and 73 | software. 74 | -------------------------------------------------------------------------------- /docs/source/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ExaWorks/SDK/a093ccedf52ef3161127ed8aa981d4f82ef75651/docs/source/favicon.ico -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ExaWorks: Software Development Kit 2 | ================================== 3 | 4 | .. image:: https://readthedocs.org/projects/exaworkssdk/badge/?version=latest 5 | :target: http://exaworkssdk.readthedocs.io/en/latest/?badge=latest 6 | :alt: Documentation Status 7 | 8 | ExaWorks **Software Development Kit (SDK)** offers: (1) packaging for a curated 9 | set of workflow software systems; (2) testing of those systems on a number of 10 | high performance computing (HPC) platforms managed by the USA Department of 11 | Energy (DoE); and (3) tutorials about coding workflow applications with those 12 | workflow systems on DoE HPC platforms. 13 | 14 | Exaworks SDK supports the workflows needs of diverse users, administrators, and 15 | developers. It enables teams to produce scalable and portable workflows for a 16 | wide range of exascale applications. SDK does not replace the many workflow 17 | solutions already deployed and used by scientists, but rather it provides a 18 | packaged, tested and documented collection of community-identified components 19 | that can be leveraged by users. SDK contributes to enabling a sustainable 20 | software infrastructure for workflows, supporting diverse scientific communities 21 | on a variety of DoE HPC platforms. 22 | 23 | Currently, ExaWorks SDK offers Docker containers and `Spack 24 | `_ packages for `Flux 25 | `_, `Parsl `_, `PSI/J 26 | `_, `RADICAL-Cybertools 27 | `_, and `Swift/T 28 | `_. Each package is deployed and tested on a 29 | growing number of DoE HPC platforms. Applications teams can draw from SDK's 30 | tutorials, levaraging containers and packages as needed to develop application 31 | workflows. 32 | 33 | 34 | .. note:: 35 | 36 | This project is under active development. 37 | 38 | 39 | .. toctree:: 40 | :numbered: 41 | :maxdepth: 2 42 | 43 | components.rst 44 | tutorials.rst 45 | contribute.rst 46 | 47 | 48 | Indices and tables 49 | ------------------ 50 | 51 | * :ref:`genindex` 52 | * :ref:`modindex` 53 | * :ref:`search` 54 | -------------------------------------------------------------------------------- /docs/source/packaging.rst: -------------------------------------------------------------------------------- 1 | .. _chapter_packaging: 2 | 3 | ========= 4 | Packaging 5 | ========= 6 | 7 | Spack 8 | ----- 9 | 10 | Spack package creation tutorial: https://spack-tutorial.readthedocs.io/en/latest/tutorial_packaging.html 11 | 12 | Highlighted steps of the package creation 13 | 14 | 1. Spack installation :: 15 | 16 | git clone https://github.com/spack/spack.git 17 | . spack/share/spack/setup-env.sh 18 | 19 | 2. Create Spack package recipe by the template :: 20 | 21 | # spack create 22 | spack create https://github.com/radical-cybertools/radical.pilot/archive/refs/tags/v1.8.0.tar.gz 23 | # package template will be created here (all python packages starts with "py-"): 24 | # /spack/var/spack/repos/builtin/packages/py-radical-pilot/package.py 25 | 26 | 3. Notes for recipe editing 27 | 28 | a) If package dependencies are not in Spack, they should be added as well; 29 | b) Variable ``maintainers`` is a list of GitHub usernames, who are 30 | responsible to maintain a corresponding Spack package (they should 31 | consented to be maintainers); 32 | c) Function call ``version('develop', branch='devel')`` maps Spack package 33 | version ``develop`` to the corresponding GitHub branch (GitHub URL of the 34 | project should be set with ``git`` variable); 35 | d) Function call ``depends_on`` lists hard requirements, not soft 36 | preferences; 37 | e) Not supported version of the package should be described with 38 | ``deprecated=True`` set and shouldn't be removed from the recipe; 39 | 40 | 4. Submit created package recipe 41 | 42 | a) Fork Spack GitHub repository (https://github.com/spack/spack.git); 43 | b) Submit created package back to Spack as a Pull Request; 44 | 45 | Spack packages list: https://spack.readthedocs.io/en/latest/package_list.html -------------------------------------------------------------------------------- /docs/source/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx_copybutton 2 | nbsphinx 3 | ipykernel 4 | sphinx-rtd-theme 5 | pandoc 6 | ipywidgets 7 | -------------------------------------------------------------------------------- /docs/source/tutorials.rst: -------------------------------------------------------------------------------- 1 | .. _chapter_tutorials: 2 | 3 | ========= 4 | Tutorials 5 | ========= 6 | 7 | In the following, we offer a brief tutorials for how to write an `hello_world` 8 | application with each Exaworks SDK core component, how to prepare and run 9 | Exaworks SDK Docker container, and how to Exaworks SDK tests. We also offer 10 | details that might be useful for developers that would like to contribute to 11 | Exaworks SDK. 12 | 13 | 14 | Running the Tutorials 15 | --------------------- 16 | 17 | Tutorials can be run via our self-contained Docker container or independently. 18 | When run independently, the user has to setup a suitable running environment for 19 | each tutorial. That can be technically demanding and it requires referring to 20 | the documentation site of each component. 21 | 22 | To run the tutorials in the ExaWorks SDK Docker container: 23 | 24 | 1. clone the ExaWorks SDK repository: 25 | 26 | ``` 27 | git clone https://github.com/ExaWorks/SDK.git 28 | ``` 29 | 30 | 2. Follow the instructions in `SDK/docker/tutorials/README.md `_, choosing one 31 | of the three methods A, B or C to execute your container. Note that if you 32 | want to run the RADICAL-Cybertools tutorial, you will have to chose either B 33 | or C. 34 | 3. After following the instructions, you will be given a URI to cut and paste in 35 | your browser to access to the Jupyter Notebook server that is running in the 36 | SDK container. 37 | 4. Load and execute each tutorial in the Jupyter Notebook server on your 38 | browser. 39 | 5. Once finished, stop the SDK container and, in case, the MongoDB container, 40 | you started to execute the RADICAL-Cybertools tutorial. 41 | 42 | 43 | 44 | SDK Tutorials 45 | ------------- 46 | 47 | .. * `Flux `_ 48 | .. * `Parsl `_ 49 | .. * `PSI/J `_ 50 | .. * `RADICAL-Cybertools `_ 51 | .. * `Swift/T `_ 52 | 53 | .. toctree:: 54 | :maxdepth: 2 55 | :glob: 56 | 57 | tutorials/* 58 | 59 | -------------------------------------------------------------------------------- /docs/source/tutorials/images/entk-pst-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ExaWorks/SDK/a093ccedf52ef3161127ed8aa981d4f82ef75651/docs/source/tutorials/images/entk-pst-model.png -------------------------------------------------------------------------------- /docs/source/tutorials/images/parsl_workflow.svg: -------------------------------------------------------------------------------- 1 | Select Next TasksSimulate(Re-)TrainInfer -------------------------------------------------------------------------------- /docs/source/tutorials/images/pi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ExaWorks/SDK/a093ccedf52ef3161127ed8aa981d4f82ef75651/docs/source/tutorials/images/pi.png -------------------------------------------------------------------------------- /docs/source/tutorials/images/psij_job_states.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ExaWorks/SDK/a093ccedf52ef3161127ed8aa981d4f82ef75651/docs/source/tutorials/images/psij_job_states.png -------------------------------------------------------------------------------- /docs/source/tutorials/images/psij_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ExaWorks/SDK/a093ccedf52ef3161127ed8aa981d4f82ef75651/docs/source/tutorials/images/psij_overview.png -------------------------------------------------------------------------------- /docs/source/tutorials/parsl-rp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "6de66e16", 6 | "metadata": {}, 7 | "source": [ 8 | "# Parsl and RADICAL-Pilot Integration\n", 9 | "\n", 10 | "RADICAL-Pilot (RP) is a runtime system that enables the execution of heterogeneous (funtions and executables) MPI workloads on heterogeneous (GPUs and CPUs) HPC resources. The integration of Parsl and RP allows RP to benefit from Parsl flexible programming model and its workflow management capabilities to build dynamic workflows. Additionally, RadicalPilotExecutor benefits Parsl by offering the heterogeneous runtime capabilities of RP to support many MPI computations more efficiently." 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "4a1580ee", 16 | "metadata": {}, 17 | "source": [ 18 | "For this tutorial we are required to update the existing default Parsl package with Parsl that has the integration files (Parsl-RP integration will be relased in Parsl soon)." 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "id": "ba1877d8", 25 | "metadata": { 26 | "ExecuteTime": { 27 | "end_time": "2022-11-22T12:17:48.357841Z", 28 | "start_time": "2022-11-22T12:17:47.458237Z" 29 | } 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "# remove the exisitng Parsl from conda\n", 34 | "!conda remove --force parsl -y\n", 35 | "\n", 36 | "# install a specific Parsl version\n", 37 | "!pip install git+https://github.com/AymenFJA/parsl.git@master" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "a98b8cb5", 43 | "metadata": {}, 44 | "source": [ 45 | "Next we locate the installed `nwchem` executable in our environment. If it is not available, we installed it from conda-forge into the local Python environment." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "84004f91", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "nwchem_path = !which nwchem\n", 56 | "\n", 57 | "if not nwchem_path:\n", 58 | " import sys\n", 59 | " !conda install --yes --prefix {sys.prefix} -c conda-forge nwchem openmpi\n", 60 | " nwchem_path = !which nwchem\n", 61 | "\n", 62 | "nwchem = nwchem_path[0]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "id": "bbf6a6d1", 68 | "metadata": {}, 69 | "source": [ 70 | "## Example: MPI NWChem Workflow\n", 71 | "\n", 72 | "The following example application shows the execution of MP2 geometry optimization followed by a CCSD(T) energy evaluation at the converged geometry. A Dunning correlation-consistent triple-zeta basis is used. The default of Cartesian basis functions must be overridden using the keyword spherical on the BASIS directive. The 1s core orbitals are frozen in both the MP2 and coupled-cluster calculations (note that these must separately specified)." 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "id": "52f78a41", 78 | "metadata": {}, 79 | "source": [ 80 | "First, we need to write the `NWChem` example to a file so that we can use it as an input for the `NWChem` executable." 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "id": "6980a500", 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "import os\n", 91 | "\n", 92 | "input = \"\"\"\n", 93 | "start n2\n", 94 | "\n", 95 | "geometry\n", 96 | " symmetry d2h\n", 97 | " n 0 0 0.542\n", 98 | "end\n", 99 | "\n", 100 | "basis spherical\n", 101 | " n library cc-pvtz\n", 102 | "end\n", 103 | "\n", 104 | "mp2\n", 105 | " freeze core\n", 106 | "end\n", 107 | "\n", 108 | "task mp2 optimize\n", 109 | "\n", 110 | "ccsd\n", 111 | " freeze core\n", 112 | "end\n", 113 | "\n", 114 | "task ccsd(t)\n", 115 | "\"\"\"\n", 116 | "nwchem_input = '{0}/{1}'.format(os.getcwd(), 'mp2_optimization.nw')\n", 117 | "with open(nwchem_input,'w+') as f:\n", 118 | " f.writelines(input)" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "id": "39d40bda", 124 | "metadata": {}, 125 | "source": [ 126 | "Now, we import the Parsl and RP Python modules in our application, alongside the RadicalPilotExecutor (RPEX) from Parsl" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "id": "85926ebe-d497-43e6-8e9a-f83714471872", 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "%env RADICAL_REPORT_ANIME=FALSE" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "id": "77692350", 143 | "metadata": { 144 | "ExecuteTime": { 145 | "end_time": "2022-11-22T12:17:57.635805Z", 146 | "start_time": "2022-11-22T12:17:57.340827Z" 147 | } 148 | }, 149 | "outputs": [], 150 | "source": [ 151 | "import parsl\n", 152 | "\n", 153 | "from parsl.config import Config\n", 154 | "from parsl.executors import RadicalPilotExecutor" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "id": "1290100d", 160 | "metadata": {}, 161 | "source": [ 162 | "`RadicalPilotExecutor` is capable of executing both functions and executables concurrently. The functions execution layer is based on the manager-worker paradigm. The managers are responsible for managing a set of workers and can execute function tasks as well. In contrast, the workers are only responsible for the function tasks execution. The manager-worker paradigm requires a set of input parameters for resource distribution, such as:\n", 163 | "1. Number of managers and workers per node\n", 164 | "2. Number of ranks per manager and worker.\n", 165 | "3. Number of nodes per manager and worker.\n", 166 | "4. Etc.\n", 167 | "\n", 168 | "In order to specify this information, we use a configuration class `ResourceConfig` that describes these parameters and pass it to `RadicalPilotExecutor`. In the cell below, we ask `RadicalPilotExecutor` to allocate 4 cores for all tasks." 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "id": "fde1d4f1", 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "# we ask Parsl to start the executor locally with 4 cores\n", 179 | "config = Config(\n", 180 | " executors=[RadicalPilotExecutor(resource='local.localhost_test', access_schema='local',\n", 181 | " project='', partition='', walltime=30, cores=4, bulk_mode=True)])\n", 182 | "\n", 183 | "parsl.load(config)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "id": "6a6f2135", 189 | "metadata": {}, 190 | "source": [ 191 | "Create a simple Parsl `@bash_app` to invoke the `NWChem` task. The `bash_app` requires the type of the task and the number of `ranks` on which to run. In this case, the type of the task is `MPI` (i.e., we set `use_mpi` explicitly), and the number of `ranks` (processes) is 2, where each rank takes 1 core.\n", 192 | "\n", 193 | "Once the `bash_app` (executable task) is invoked, the `RadicalPilotExecutor` submits the task to the runtime system and wait for them to be executed. `RadicalPilotExecutor` creates a designated `sandbox` folder that contains the tasks and their `stdout/stderr` files." 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": null, 199 | "id": "ad3e90a9", 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "@parsl.bash_app\n", 204 | "def nwchem_mp2_optimization(ranks=2, use_mpi=True):\n", 205 | "\n", 206 | " return '{0} {1}'.format(nwchem, nwchem_input)\n", 207 | "\n", 208 | "# invoke the nwchem_mp2_optimization\n", 209 | "future = nwchem_mp2_optimization()\n", 210 | "\n", 211 | "# wait for the results of the NWChem task.\n", 212 | "if future.result() == 0:\n", 213 | " print('Parsl task {0} finished'.format(future.tid))\n", 214 | " \n", 215 | " # RP has a different task id than Parsl (task.id)\n", 216 | " task_id = str(future.tid).zfill(6)\n", 217 | "\n", 218 | " # RP tasks output located in the sandbox folder\n", 219 | " task_path = '{0}/radical.pilot.sandbox/{1}/pilot.0000/task.{2}/task.{2}.out'.format(os.path.expanduser('~'),\n", 220 | " config.executors[0].session.uid, task_id)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "id": "34d6ebb7", 226 | "metadata": {}, 227 | "source": [ 228 | "print task output from the task file" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "id": "fa510510", 235 | "metadata": {}, 236 | "outputs": [], 237 | "source": [ 238 | "task_out = open(task_path, 'r').readlines()\n", 239 | "print(''.join(task_out))" 240 | ] 241 | }, 242 | { 243 | "cell_type": "markdown", 244 | "id": "e198583e", 245 | "metadata": {}, 246 | "source": [ 247 | "Finally, shutdown the executor, otherwise it will always stays ready to get more tasks" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "id": "6d044d15-3678-4d0f-9e36-d4bd50664689", 254 | "metadata": {}, 255 | "outputs": [], 256 | "source": [ 257 | "config.executors[0].shutdown()" 258 | ] 259 | } 260 | ], 261 | "metadata": { 262 | "kernelspec": { 263 | "display_name": "Python 3 (ipykernel)", 264 | "language": "python", 265 | "name": "python3" 266 | }, 267 | "language_info": { 268 | "codemirror_mode": { 269 | "name": "ipython", 270 | "version": 3 271 | }, 272 | "file_extension": ".py", 273 | "mimetype": "text/x-python", 274 | "name": "python", 275 | "nbconvert_exporter": "python", 276 | "pygments_lexer": "ipython3", 277 | "version": "3.9.13" 278 | }, 279 | "varInspector": { 280 | "cols": { 281 | "lenName": 16, 282 | "lenType": 16, 283 | "lenVar": 40 284 | }, 285 | "kernels_config": { 286 | "python": { 287 | "delete_cmd_postfix": "", 288 | "delete_cmd_prefix": "del ", 289 | "library": "var_list.py", 290 | "varRefreshCmd": "print(var_dic_list())" 291 | }, 292 | "r": { 293 | "delete_cmd_postfix": ") ", 294 | "delete_cmd_prefix": "rm(", 295 | "library": "var_list.r", 296 | "varRefreshCmd": "cat(var_dic_list()) " 297 | } 298 | }, 299 | "types_to_exclude": [ 300 | "module", 301 | "function", 302 | "builtin_function_or_method", 303 | "instance", 304 | "_Feature" 305 | ], 306 | "window_display": false 307 | } 308 | }, 309 | "nbformat": 4, 310 | "nbformat_minor": 5 311 | } 312 | -------------------------------------------------------------------------------- /docs/source/tutorials/psij.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# PSI/J-Python Getting Started Tutorial" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "\n", 15 | "\n", 16 | "PSI/J (Portable Submission Interface for Jobs), is an abstraction layer over cluster job schedulers. It allows your application to be written in a way that is (mostly) independent of the cluster(s) where it runs. It is a language agnostic specification. PSI/J-Python is a Python implementation of PSI/J.\n", 17 | "\n" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "### Installation" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "%pip install git+https://github.com/ExaWorks/psij-python.git >/dev/null 2>&1\n", 34 | "%pip show psij-python" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "## Overview \n", 42 | "\n", 43 | "When running a job, there are a number of things to specify:\n", 44 | "- What is to be run, such as executable, arguments, environment, etc. ([JobSpec](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.job_spec.JobSpec))\n", 45 | "- What resources are needed by the job, such as the number of nodes ([ResourceSpec](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.resource_spec.ResourceSpecV1))\n", 46 | "- Various miscellaneous properties, such as the queue to submit the job to ([JobAttributes](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.job_attributes.JobAttributes))\n", 47 | "- The mechanism through which to run the job, such as local/exec, SLURM, PBS, etc. ([JobExecutor](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.job_executor.JobExecutor))\n", 48 | "\n", 49 | "We also need an object to keep track of all this information, as well as the state of the execution. This object is an instance of a [Job](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.job.Job)." 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "### Setup\n", 57 | "\n", 58 | "Before we start, let us create a separate directory so that we don't ovewrite each others' files" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "import os\n", 68 | "from tempfile import mkdtemp\n", 69 | "\n", 70 | "os.makedirs('./userdirs', exist_ok=True)\n", 71 | "workdir = mkdtemp(prefix='userdir-', dir='./userdirs')\n", 72 | "os.chdir(workdir)\n", 73 | "print(workdir)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "### Basic Usage\n", 81 | "Without further ado, let's create a simple job:" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "from pathlib import Path\n", 91 | "from psij import Job, JobSpec\n", 92 | "\n", 93 | "job = Job(JobSpec(executable='/bin/date', stdout_path=Path('the-date.txt')))" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "Easy. We created a job that runs `/bin/date` and stores the output in `the-date.txt`. \n", 101 | "Now we need to run it. In order to do so, we need an *executor* that knows how to run jobs. We will use a simple fork/exec based executor named `local`. On a real cluster, we would use something like `SLURM` or `LSF`, but we are not doing this on a real cluster. However, I will note here that in most cases, simply changing `local` to the name of the scheduler used by the cluster would be sufficient to run the job through the cluster scheduler." 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "from psij import JobExecutor\n", 111 | "\n", 112 | "executor = JobExecutor.get_instance('local')" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "We can now tell the executor to run our job" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "executor.submit(job)" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "The [submit()](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.job_executor.JobExecutor.submit) method **starts** the job asynchronously.\n", 136 | "We would now like to see the result. However, before we can do so, we must ensure that the job has actually finished running. We can do so by [waiting](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.job.Job.wait) for it:" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "job.wait()" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "The `wait()` method returns the [JobStatus](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.JobStatus). Since nothing can possibly go wrong, we will assume that the job completed successfully and that there is no need to check the status to confirm it. Now, we can finally read the output" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "with open('the-date.txt') as f:\n", 162 | " print(f.read())" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "### Multiple Jobs\n", 170 | "Our executor is stateless. That means that we can submit as many jobs as we want to it. That's in theory. In practice, computers have limited resources and there are only so many concurrent jobs that we can run, but hopefully we won't hit those limits today." 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "jobs = []\n", 180 | "for i in range(10):\n", 181 | " job = Job(\n", 182 | " JobSpec(\n", 183 | " executable='/bin/echo', \n", 184 | " arguments=['Hello from job %s' % i],\n", 185 | " stdout_path=Path('hello-%s.txt' % i)\n", 186 | " )\n", 187 | " )\n", 188 | " executor.submit(job)\n", 189 | " jobs.append(job)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "metadata": {}, 195 | "source": [ 196 | "If these jobs weren't so short, they would now be running in parallel. In fact, why not start a longer job:" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": null, 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "long_job = Job(JobSpec(executable='/bin/sleep', arguments=['600']))\n", 206 | "executor.submit(long_job)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "markdown", 211 | "metadata": {}, 212 | "source": [ 213 | "Back to our previous jobs. In order to read their outputs, we must, again, ensure that they are done" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "for i in range(10):\n", 223 | " jobs[i].wait()\n", 224 | " with open('hello-%s.txt' % i) as f:\n", 225 | " print(f.read())" 226 | ] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "metadata": {}, 231 | "source": [ 232 | "What about our long job?" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "print(long_job.status)" 242 | ] 243 | }, 244 | { 245 | "cell_type": "markdown", 246 | "metadata": {}, 247 | "source": [ 248 | "Still running. The time shows the instant when the job switched to `ACTIVE` state. Moving on..." 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": {}, 254 | "source": [ 255 | "### Multi-process Jobs\n", 256 | "So far we've run multiple independent jobs. But what if we wanted to run multiple copies of one job, presumably on multiple compute nodes (this is a Docker container, but we can pretend)?\n", 257 | "We could tell PSI/J to do this using [ResourceSpecV1](https://exaworks.org/psij-python/#docs/.generated/psij.html/#psij.resource_spec.ResourceSpecV1). We also need to tell PSI/J to start our job a bit differently, so we'll make a short detour to talk about launchers.\n", 258 | "\n", 259 | "Once a job's resources are allocated, a typical job scheduler will launch our job on one of the allocated compute nodes. Then, we'd invoke something like `mpirun` or `srun`, etc. to start all the job copies on the allocated resources. By default, PSI/J uses a custom launcher named `single`, which simply starts a single copy of the job on the lead node of the job. If we wanted to see multiple copies of the job without any of the fancy features offered by `mpirun` or `srun`, we could use PSI/J's `multiple` launcher, which we will do below." 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": null, 265 | "metadata": {}, 266 | "outputs": [], 267 | "source": [ 268 | "from psij import ResourceSpecV1\n", 269 | "\n", 270 | "mjob = Job(\n", 271 | " JobSpec(\n", 272 | " executable='/bin/date', \n", 273 | " stdout_path=Path('multi-job-out.txt'),\n", 274 | " resources=ResourceSpecV1(process_count=4),\n", 275 | " launcher=\"multiple\"\n", 276 | " )\n", 277 | " )" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": {}, 283 | "source": [ 284 | "We informed PSI/J that we need four copies of our job. On a real scheduler, we could also request that these copies be distributed on multiple compute nodes, but, on this VM, we only have one such compute node, so we shoudn't bother." 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": null, 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [ 293 | "executor.submit(mjob)\n", 294 | "mjob.wait()\n", 295 | "with open('multi-job-out.txt') as f:\n", 296 | " print(f.read())" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": {}, 302 | "source": [ 303 | "### MPI Jobs\n", 304 | "\n", 305 | "The previous example ran a multi-process job, which has its use. It is more likely, however, to want to run an MPI job. Assuming that the system has some form of MPI installed, which this Docker container has, and which comes with some generic `mpirun` tool, we can instruct PSI/J to launch MPI jobs. And, as the previous sentence hints, it may be as simple as changing our launcher from `multiple` to `mpirun`, which it is.\n", 306 | "\n", 307 | "But before that, we need a simple MPI executable. " 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": null, 313 | "metadata": {}, 314 | "outputs": [], 315 | "source": [ 316 | "%%bash\n", 317 | "cat <hello.c\n", 318 | "#include \n", 319 | "#include \n", 320 | "\n", 321 | "void main(int argc, char **argv) {\n", 322 | " int rank;\n", 323 | " MPI_Init(&argc, &argv);\n", 324 | " MPI_Comm_rank(MPI_COMM_WORLD, &rank);\n", 325 | " \n", 326 | " printf(\"Hello from rank %d\\n\", rank);\n", 327 | " \n", 328 | " MPI_Finalize();\n", 329 | "}\n", 330 | "EOF" 331 | ] 332 | }, 333 | { 334 | "cell_type": "markdown", 335 | "metadata": {}, 336 | "source": [ 337 | "Which we need to compile" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": null, 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "!mpicc hello.c -o hello" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "And now we can construct our job" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": null, 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [ 362 | "mpi_job = Job(\n", 363 | " JobSpec(\n", 364 | " executable='hello', \n", 365 | " stdout_path=Path('mpi-job-out.txt'),\n", 366 | " resources=ResourceSpecV1(process_count=4),\n", 367 | " launcher=\"mpirun\"\n", 368 | " )\n", 369 | " )" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "metadata": {}, 375 | "source": [ 376 | "... and, as usual, wait for it and display the output" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": null, 382 | "metadata": {}, 383 | "outputs": [], 384 | "source": [ 385 | "executor.submit(mpi_job)\n", 386 | "mpi_job.wait()\n", 387 | "with open('mpi-job-out.txt') as f:\n", 388 | " print(f.read())" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "And the long running job?" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": null, 401 | "metadata": {}, 402 | "outputs": [], 403 | "source": [ 404 | "print(long_job)" 405 | ] 406 | }, 407 | { 408 | "cell_type": "markdown", 409 | "metadata": {}, 410 | "source": [ 411 | "Soon, soon..." 412 | ] 413 | }, 414 | { 415 | "cell_type": "markdown", 416 | "metadata": {}, 417 | "source": [ 418 | "### Callbacks\n", 419 | "\n", 420 | "Examples above are more or less synchronous, in that we use `wait()` to suspend the current thread until a job completes. In real life scenarios where scalability is needed, we would use callbacks. Let's implement a quick map/reduce workflow. We'll Monte Carlo calculate π using a map-reduce like algorithm.\n", 421 | "\n", 422 | "The basic idea is to generate some random points on a square that encloses one quadrant of a circle. \n", 423 | "\n", 424 | "\n", 425 | "\n", 426 | "Some points will fall outside the circle and some inside. As the number of points grows, the ratio of points inside the circle vs points inside the full square (total points) will be proportional to the ratio of their areas: \n", 427 | "\n", 428 | "Ncircle / Ntotal ≈ Acircle / Asquare = (πr2 / 4) / r2\n", 429 | "\n", 430 | "Hence\n", 431 | "\n", 432 | "π = 4 Ncircle / Ntotal\n", 433 | "\n", 434 | "We'll start with some boilerplate, the number of iterations, and the radius of the circle" 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "execution_count": null, 440 | "metadata": {}, 441 | "outputs": [], 442 | "source": [ 443 | "from threading import Lock\n", 444 | "from psij import JobState\n", 445 | "import math\n", 446 | "\n", 447 | "N = 100\n", 448 | "R = 1000" 449 | ] 450 | }, 451 | { 452 | "cell_type": "markdown", 453 | "metadata": {}, 454 | "source": [ 455 | "Then, we'll define a class that keeps track of our points and calculates π once we have all the points in, and we'll create an instance of it to hold actual results." 456 | ] 457 | }, 458 | { 459 | "cell_type": "code", 460 | "execution_count": null, 461 | "metadata": {}, 462 | "outputs": [], 463 | "source": [ 464 | "class Results:\n", 465 | " def __init__(self):\n", 466 | " self.n = 0\n", 467 | " self.inside = 0\n", 468 | " self._lock = Lock()\n", 469 | " \n", 470 | " def point_received(self, x, y):\n", 471 | " with self._lock:\n", 472 | " self.n += 1\n", 473 | " if math.sqrt(x * x + y * y) < R:\n", 474 | " self.inside += 1\n", 475 | " if self.n == N:\n", 476 | " print(\"π is %s\" % (float(self.inside) / self.n * 4))\n", 477 | "\n", 478 | "results = Results()" 479 | ] 480 | }, 481 | { 482 | "cell_type": "markdown", 483 | "metadata": {}, 484 | "source": [ 485 | "Then, we'll define a callback function that gets invoked every time a job changes status, and have it read the output and pass it to the `results` instance. The output will be in the form `x y`" 486 | ] 487 | }, 488 | { 489 | "cell_type": "code", 490 | "execution_count": null, 491 | "metadata": {}, 492 | "outputs": [], 493 | "source": [ 494 | "def callback(job, status):\n", 495 | " if status.state == JobState.COMPLETED:\n", 496 | " with open(job.spec.stdout_path) as f:\n", 497 | " line = f.read().strip()\n", 498 | " tokens = line.split()\n", 499 | " results.point_received(int(tokens[0]), int(tokens[1]))" 500 | ] 501 | }, 502 | { 503 | "cell_type": "markdown", 504 | "metadata": {}, 505 | "source": [ 506 | "Unlike in previous cases, we now need to check the state of the job. That is because the full lifecycle of the job includes states such as `QUEUED` and `ACTIVE`, and the callback is invoked on all state changes.\n", 507 | "\n", 508 | "Finally, we can create and submit our jobs" 509 | ] 510 | }, 511 | { 512 | "cell_type": "code", 513 | "execution_count": null, 514 | "metadata": {}, 515 | "outputs": [], 516 | "source": [ 517 | "for i in range(N):\n", 518 | " job = Job(JobSpec('echo', '/bin/bash', \n", 519 | " ['-c', 'echo $((RANDOM%{})) $((RANDOM%{}))'.format(R, R)], \n", 520 | " stdout_path=Path('pi-x-y-%s.txt' % i)))\n", 521 | " job.set_job_status_callback(callback)\n", 522 | " executor.submit(job)" 523 | ] 524 | }, 525 | { 526 | "cell_type": "markdown", 527 | "metadata": {}, 528 | "source": [ 529 | "Sure!\n", 530 | "Notice that the main thread is free as soon as the last job is submitted." 531 | ] 532 | }, 533 | { 534 | "cell_type": "markdown", 535 | "metadata": {}, 536 | "source": [ 537 | "That's about it for this tutorial. Oh, the long running job should be done now." 538 | ] 539 | }, 540 | { 541 | "cell_type": "code", 542 | "execution_count": null, 543 | "metadata": {}, 544 | "outputs": [], 545 | "source": [ 546 | "print(long_job)" 547 | ] 548 | }, 549 | { 550 | "cell_type": "markdown", 551 | "metadata": {}, 552 | "source": [ 553 | "If not, we can stop it" 554 | ] 555 | }, 556 | { 557 | "cell_type": "code", 558 | "execution_count": null, 559 | "metadata": {}, 560 | "outputs": [], 561 | "source": [ 562 | "long_job.cancel()" 563 | ] 564 | }, 565 | { 566 | "cell_type": "markdown", 567 | "metadata": {}, 568 | "source": [ 569 | "OK, now we're really done. So it's clean up time. And you know what they say, if all you have is a hammer..." 570 | ] 571 | }, 572 | { 573 | "cell_type": "code", 574 | "execution_count": null, 575 | "metadata": {}, 576 | "outputs": [], 577 | "source": [ 578 | "os.chdir('../../')\n", 579 | "cleanup_job = Job(\n", 580 | " JobSpec(\n", 581 | " executable='/bin/rm',\n", 582 | " arguments=['-rf', workdir],\n", 583 | " directory=Path('.')\n", 584 | " )\n", 585 | ")\n", 586 | "executor.submit(cleanup_job)" 587 | ] 588 | }, 589 | { 590 | "cell_type": "markdown", 591 | "metadata": {}, 592 | "source": [ 593 | "Thank you!" 594 | ] 595 | } 596 | ], 597 | "metadata": { 598 | "kernelspec": { 599 | "display_name": "Python 3 (ipykernel)", 600 | "language": "python", 601 | "name": "python3" 602 | }, 603 | "language_info": { 604 | "codemirror_mode": { 605 | "name": "ipython", 606 | "version": 3 607 | }, 608 | "file_extension": ".py", 609 | "mimetype": "text/x-python", 610 | "name": "python", 611 | "nbconvert_exporter": "python", 612 | "pygments_lexer": "ipython3", 613 | "version": "3.9.16" 614 | }, 615 | "vscode": { 616 | "interpreter": { 617 | "hash": "76009bf859c7381f9309fe432e50ba00bc4a43c07b3a3292e8eda4f7573404cf" 618 | } 619 | } 620 | }, 621 | "nbformat": 4, 622 | "nbformat_minor": 4 623 | } 624 | -------------------------------------------------------------------------------- /docs/source/tutorials/swift-t.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "21938011", 6 | "metadata": {}, 7 | "source": [ 8 | "# Swift/T Getting Started Tutorial" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "id": "b48b9ef3", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "! git clone -b main --single-branch https://github.com/ExaWorks/Tutorial.git ./sdk-examples" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "id": "646bc4ab", 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "import os\n", 29 | "\n", 30 | "path = os.environ['PATH']\n", 31 | "os.environ['PATH'] = \"%s:/tmp/swift-t-install/stc/bin:/tmp/swift-t-install/turbine/bin\" % path" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "id": "82a3570e", 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "def swift_t(code):\n", 42 | " import subprocess\n", 43 | " subprocess.run([\"swift-t\", \"-t -oversubscribe -E\", code])" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "id": "2f0c3530", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "swift_t(\"trace(42);\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "id": "ced35680", 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "def swift_t_file(directory, filename):\n", 64 | " import os, subprocess\n", 65 | " original = os.getcwd()\n", 66 | " os.chdir(directory)\n", 67 | " try:\n", 68 | " subprocess.run([\"swift-t -t -oversubscribe\", filename])\n", 69 | " except:\n", 70 | " pass\n", 71 | " os.chdir(original)" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "id": "79709bcd", 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "swift_t_file(\"sdk-examples/2-workflow-dl-swift/01-hello\", \"hello.swift\")" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "id": "24798e1a", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "swift_t_file(\"sdk-examples/2-workflow-dl-swift/02-loop\", \"loop.swift\")" 92 | ] 93 | } 94 | ], 95 | "metadata": { 96 | "kernelspec": { 97 | "display_name": "Python 3 (ipykernel)", 98 | "language": "python", 99 | "name": "python3" 100 | }, 101 | "language_info": { 102 | "codemirror_mode": { 103 | "name": "ipython", 104 | "version": 3 105 | }, 106 | "file_extension": ".py", 107 | "mimetype": "text/x-python", 108 | "name": "python", 109 | "nbconvert_exporter": "python", 110 | "pygments_lexer": "ipython3", 111 | "version": "3.9.16" 112 | } 113 | }, 114 | "nbformat": 4, 115 | "nbformat_minor": 5 116 | } 117 | --------------------------------------------------------------------------------