├── .gitattributes ├── .github └── workflows │ ├── ebrains-push.yml │ ├── integration-test-unicore10.yml │ ├── integration-test-unicore9.yml │ ├── lint.yaml │ ├── pypi-publish.yml │ └── unit-test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CHANGES.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── docs ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── _static │ └── logo-unicore.png │ ├── authentication.rst │ ├── basic_usage.rst │ ├── conf.py │ ├── dask.rst │ ├── index.rst │ ├── license.rst │ ├── port_forwarding.rst │ └── uftp.rst ├── pyproject.toml ├── pyunicore ├── __init__.py ├── _version.py ├── cli │ ├── __init__.py │ ├── base.py │ ├── exec.py │ ├── info.py │ ├── io.py │ └── main.py ├── client.py ├── credentials.py ├── cwl │ ├── __init__.py │ ├── cwlconverter.py │ └── cwltool.py ├── dask.py ├── forwarder.py ├── helpers │ ├── __init__.py │ ├── _api_object.py │ ├── connection │ │ ├── __init__.py │ │ ├── registry.py │ │ └── site.py │ ├── jobs │ │ ├── __init__.py │ │ ├── data.py │ │ ├── description.py │ │ └── resources.py │ └── workflows │ │ ├── __init__.py │ │ ├── activities │ │ ├── __init__.py │ │ ├── activity.py │ │ ├── job.py │ │ ├── loops │ │ │ ├── __init__.py │ │ │ ├── _loop.py │ │ │ ├── body.py │ │ │ ├── for_loop.py │ │ │ ├── repeat_until_loop.py │ │ │ └── while_loop.py │ │ └── modify_variable.py │ │ ├── description.py │ │ ├── transition.py │ │ └── variable.py └── uftp │ ├── __init__.py │ ├── uftp.py │ ├── uftpfs.py │ ├── uftpfuse.py │ └── uftpmountfs.py ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── cwldocs │ ├── array-inputs.cwl │ ├── array-inputs.params │ ├── directoryinput.cwl │ ├── directoryinput.params │ ├── echo.cwl │ ├── echo.params │ ├── fetching_data_tool.cwl │ ├── fetching_data_tool.params │ ├── fileinput.cwl │ ├── fileinput.params │ ├── fileinput_remote.cwl │ └── fileinput_remote.params ├── integration │ ├── cli │ │ ├── jobs │ │ │ └── date.u │ │ ├── preferences │ │ ├── test_base.py │ │ ├── test_exec.py │ │ └── test_io.py │ ├── files │ │ ├── script.sh │ │ └── workflow1.json │ ├── test_auth.py │ ├── test_basic.py │ ├── test_registry.py │ ├── test_storage.py │ └── test_workflow.py ├── testing │ ├── __init__.py │ ├── contexts.py │ └── pyunicore.py └── unit │ ├── cli │ ├── preferences │ ├── test_base.py │ ├── test_io.py │ └── test_main.py │ ├── helpers │ ├── connection │ │ ├── test_registry_helper.py │ │ └── test_site.py │ ├── jobs │ │ ├── test_data.py │ │ └── test_job_description.py │ ├── test_api_object.py │ └── workflows │ │ ├── activities │ │ ├── loops │ │ │ ├── conftest.py │ │ │ ├── test_for_loop.py │ │ │ ├── test_repeat_until_loop.py │ │ │ └── test_while_loop.py │ │ ├── test_activity.py │ │ ├── test_job_activity.py │ │ └── test_modify_variable.py │ │ ├── test_transition.py │ │ ├── test_variable.py │ │ └── test_workflow_description.py │ ├── test_cwl1.py │ ├── test_jwt_tokens.py │ ├── test_uftpfs.py │ ├── test_utils.py │ └── token.txt └── versioneer.py /.gitattributes: -------------------------------------------------------------------------------- 1 | pyunicore/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.github/workflows/ebrains-push.yml: -------------------------------------------------------------------------------- 1 | name: Mirror to EBRAINS 2 | 3 | on: 4 | push: 5 | branches: [ dev ] 6 | 7 | jobs: 8 | to_ebrains: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: syncmaster 12 | uses: wei/git-sync@v3 13 | with: 14 | source_repo: "HumanBrainProject/pyunicore" 15 | source_branch: "dev" 16 | destination_repo: "https://ghpusher:${{ secrets.EBRAINS_GITLAB_ACCESS_TOKEN }}@gitlab.ebrains.eu/unicore/pyunicore.git" 17 | destination_branch: "dev" 18 | - name: synctags 19 | uses: wei/git-sync@v3 20 | with: 21 | source_repo: "HumanBrainProject/pyunicore" 22 | source_branch: "refs/tags/*" 23 | destination_repo: "https://ghpusher:${{ secrets.EBRAINS_GITLAB_ACCESS_TOKEN }}@gitlab.ebrains.eu/unicore/pyunicore.git" 24 | destination_branch: "refs/tags/*" 25 | -------------------------------------------------------------------------------- /.github/workflows/integration-test-unicore10.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs integration tests using 2 | # the unicore-testing-all Docker container 3 | 4 | name: Integration tests vs UNICORE 10 5 | 6 | on: 7 | workflow_dispatch: 8 | push: 9 | pull_request: 10 | 11 | jobs: 12 | integration-test: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | services: 17 | unicore: 18 | image: ghcr.io/unicore-eu/unicore-testing-all:10.2.0 19 | ports: 20 | - 8080:8080 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Set up Python 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: '3.x' 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | pip install setuptools wheel 32 | pip install . 33 | - name: Run tests 34 | run: | 35 | make integration-test 36 | -------------------------------------------------------------------------------- /.github/workflows/integration-test-unicore9.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs integration tests using 2 | # the unicore-testing-all Docker container 3 | 4 | name: Integration tests vs UNICORE 9 5 | 6 | on: 7 | workflow_dispatch: 8 | push: 9 | pull_request: 10 | 11 | jobs: 12 | integration-test: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | services: 17 | unicore: 18 | image: ghcr.io/unicore-eu/unicore-testing-all:9.3.1 19 | ports: 20 | - 8080:8080 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Set up Python 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: '3.x' 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | pip install setuptools wheel 32 | pip install . 33 | - name: Run tests 34 | run: | 35 | make integration-test 36 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | # This workflow runs linting packages 2 | 3 | name: Lint 4 | 5 | on: 6 | workflow_dispatch: 7 | push: 8 | pull_request: 9 | 10 | jobs: 11 | black: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Set up Python 16 | uses: actions/setup-python@v4 17 | with: 18 | python-version: '3.x' 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install black 23 | - name: Lint package 24 | run: | 25 | black \ 26 | --check \ 27 | --config pyproject.toml \ 28 | pyunicore tests 29 | flake8: 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v3 33 | - name: Set up Python 34 | uses: actions/setup-python@v4 35 | with: 36 | python-version: '3.x' 37 | - name: Install dependencies 38 | run: | 39 | python -m pip install --upgrade pip 40 | pip install \ 41 | setuptools \ 42 | flake8 \ 43 | flake8-pyproject \ 44 | flake8-blind-except \ 45 | flake8-logging-format \ 46 | flake8_module_name \ 47 | pep8-naming 48 | - name: Lint package 49 | run: | 50 | flake8 pyunicore tests 51 | pyupgrade: 52 | runs-on: ubuntu-latest 53 | steps: 54 | - uses: actions/checkout@v3 55 | - name: Set up Python 56 | uses: actions/setup-python@v4 57 | with: 58 | python-version: '3.x' 59 | - name: Install dependencies 60 | run: | 61 | python -m pip install --upgrade pip 62 | pip install pyupgrade 63 | - name: Lint package 64 | run: | 65 | pyupgrade \ 66 | --py3-plus \ 67 | --py36-plus \ 68 | $(find pyunicore tests | grep "\.py$") 69 | isort: 70 | runs-on: ubuntu-latest 71 | steps: 72 | - uses: actions/checkout@v3 73 | - name: Set up Python 74 | uses: actions/setup-python@v4 75 | with: 76 | python-version: '3.x' 77 | - name: Install dependencies 78 | run: | 79 | python -m pip install --upgrade pip 80 | pip install isort 81 | - name: Check imports order and formatting 82 | run: | 83 | isort --check --profile black --force-single-line-imports $(find pyunicore tests | grep "\.py$") 84 | -------------------------------------------------------------------------------- /.github/workflows/pypi-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a tag is created 2 | # the latest tag is used (via versioneer) 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | push: 8 | tags: 9 | - '**' 10 | 11 | jobs: 12 | deploy: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: '3.x' 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install setuptools wheel twine 26 | - name: Build and publish 27 | env: 28 | TWINE_USERNAME: __token__ 29 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 30 | run: | 31 | python setup.py sdist 32 | twine upload dist/* 33 | -------------------------------------------------------------------------------- /.github/workflows/unit-test.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs unit tests 2 | 3 | name: Unit tests 4 | 5 | on: 6 | workflow_dispatch: 7 | push: 8 | pull_request: 9 | 10 | jobs: 11 | unit-test: 12 | 13 | runs-on: ubuntu-22.04 14 | strategy: 15 | matrix: 16 | python-version: ["3.7", "3.x"] 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Python 21 | uses: actions/setup-python@v4 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install setuptools wheel 28 | pip install . 29 | pip install -r requirements-dev.txt 30 | - name: Run tests 31 | run: | 32 | make test 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.pyc 4 | *~ 5 | .project 6 | .pydevproject 7 | dist/ 8 | pyunicore.egg-info/ 9 | .settings 10 | .coverage/ 11 | .coverage 12 | build/ 13 | /.pytest_cache/ 14 | .env 15 | .vscode 16 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | fail_fast: false 2 | default_language_version: 3 | python: python3 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v4.3.0 7 | hooks: 8 | - id: check-added-large-files 9 | args: [ "--maxkb=500" ] 10 | exclude: (__pycache__, *.pyc, *.pyo, *.iml, *.xml, *.cpy) 11 | - id: check-ast 12 | - id: check-builtin-literals 13 | - id: check-case-conflict 14 | - id: check-docstring-first 15 | - id: check-json 16 | - id: check-toml 17 | - id: debug-statements 18 | - id: end-of-file-fixer 19 | - id: mixed-line-ending 20 | - id: trailing-whitespace 21 | exclude: (__pycache__, *.pyc, *.pyo, *.iml, *.xml, *.cpy) 22 | 23 | - repo: https://github.com/asottile/pyupgrade 24 | rev: v3.1.0 25 | hooks: 26 | - id: pyupgrade 27 | args: [ 28 | "--py3-plus", 29 | "--py36-plus", 30 | ] 31 | 32 | - repo: https://github.com/psf/black 33 | rev: 24.3.0 34 | hooks: 35 | - id: black 36 | args: ["--config", "pyproject.toml"] 37 | 38 | - repo: https://github.com/pycqa/flake8 39 | rev: 7.1.0 40 | hooks: 41 | - id: flake8 42 | additional_dependencies: 43 | - Flake8-pyproject 44 | # See https://github.com/DmytroLitvinov/awesome-flake8-extensions 45 | - flake8-blind-except 46 | - flake8-logging-format 47 | - flake8_module_name 48 | - pep8-naming 49 | 50 | - repo: https://github.com/pycqa/isort 51 | rev: 5.13.2 52 | hooks: 53 | - id: isort 54 | args: [--profile, black, --force-single-line-imports] 55 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.8" 13 | 14 | # Build documentation in the source/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/source/conf.py 17 | 18 | # Optionally declare the Python requirements required to build your docs 19 | python: 20 | install: 21 | - requirements: docs/requirements.txt 22 | 23 | 24 | # Optionally build your docs in additional formats such as PDF and ePub 25 | formats: 26 | - pdf 27 | - epub 28 | - htmlzip 29 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | Changelog for PyUNICORE 2 | ======================= 3 | 4 | Issue tracker: https://github.com/HumanBrainProject/pyunicore 5 | 6 | Version 1.4.0 (MMM dd, 2025) 7 | ---------------------------- 8 | - CLI: implement server-to-server transfers 9 | 10 | Version 1.3.3 (June 2, 2025) 11 | ---------------------------- 12 | - fix: UFTPFS: add workaround for bug in base class FTPFS 13 | 14 | Version 1.3.2 (May 15, 2025) 15 | ---------------------------- 16 | - fix: add "preferences" and "persistent" options to the 17 | UFTP.authenticate() function 18 | 19 | 20 | Version 1.3.1 (May 15, 2025) 21 | ---------------------------- 22 | - fix: allow single-valued extra options for the FUSE driver 23 | 24 | Version 1.3.0 (May 12, 2025) 25 | ---------------------------- 26 | - new feature: CLI: implemented more authentication options 27 | (oidc-agent, oidc-server, anonymous) 28 | - new feature: CLI: 'info' command 29 | 30 | Version 1.2.0 (Nov 08, 2024) 31 | ---------------------------- 32 | 33 | - UFTP FUSE driver: add '--read-only' option 34 | - New feature: commandline client script 'unicore' with a number 35 | of commands modeled after the 'ucc' commandline client 36 | - fix: the Job.working_dir function was hanging in case the working 37 | directory could not be created 38 | 39 | Version 1.1.1 (Oct 01, 2024) 40 | ---------------------------- 41 | - fix: type annotations did not work on Python3.7 42 | 43 | Version 1.1.0 (Sep 30, 2024) 44 | ---------------------------- 45 | - API CHANGE: new Storage.put_file() method accepting 46 | str-like or file-like data to upload to a remote destination 47 | - new feature: new pyfilesystem implementation "uftpmount" which mounts 48 | the remote directory and then accesses it via the local FS (OSFS) 49 | - fix: make sure job working directory is ready for use (fixes a 50 | potential race condition with UNICORE 10.1) 51 | 52 | Version 1.0.1 (Mar 22, 2024) 53 | ---------------------------- 54 | - fix: setting transport preferences immediately and automatically 55 | "takes effect" without requiring additional action by the 56 | user of the class 57 | 58 | Version 1.0.0 (Feb 23, 2024) 59 | ---------------------------- 60 | - after many 0.x releases in the course of the Human Brain Project, 61 | we decided to finally call it "1.0.0" 62 | - for a full list of releases, see 63 | https://pypi.org/project/pyunicore/#history 64 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) Human Brain Project, Forschungszentrum Juelich GmbH 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the names of the copyright holders nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include versioneer.py 3 | include pyunicore/_version.py 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TESTS = tests/unit 2 | INTEGRATIONTESTS = $(wildcard tests/integration/test_*.py) 3 | export PYTHONPATH := . 4 | PYTHON = python3 5 | PYTEST = pytest 6 | 7 | test: runtest 8 | 9 | integration-test: runintegrationtest 10 | 11 | .PHONY: runtest $(TESTS) runintegrationtest $(INTEGRATIONTESTS) 12 | 13 | runtest: $(TESTS) 14 | 15 | $(TESTS): 16 | @echo "\n** Running test $@" 17 | @${PYTEST} $@ 18 | 19 | runintegrationtest: $(INTEGRATIONTESTS) 20 | 21 | $(INTEGRATIONTESTS): 22 | @echo "\n** Running integration test $@" 23 | @${PYTHON} $@ 24 | 25 | clean: 26 | @find -name "*~" -delete 27 | @find -name "*.pyc" -delete 28 | @find -name "__pycache__" -delete 29 | @rm -rf build/ 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyUNICORE, a Python library for using UNICORE and UFTP 2 | 3 | This library covers the UNICORE REST API, making common tasks like 4 | file access, job submission and management, workflow submission and 5 | management more convenient, and integrating UNICORE features better 6 | with typical Python usage. 7 | 8 | The full, up-to-date documentation of the REST API can be found 9 | [here](https://unicore-docs.readthedocs.io/en/latest/user-docs/rest-api) 10 | 11 | In addition, this library contains code for using UFTP (UNICORE FTP) 12 | for filesystem mounts with FUSE, a UFTP driver for 13 | [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) 14 | and a UNICORE implementation of a 15 | [Dask Cluster](https://distributed.dask.org/en/stable/) 16 | 17 | This project has received funding from the European Union’s 18 | Horizon 2020 Framework Programme for Research and Innovation under the 19 | Specific Grant Agreement Nos. 720270, 785907 and 945539 20 | (Human Brain Project SGA 1, 2 and 3) 21 | 22 | See LICENSE file for licensing information 23 | 24 | ## Documentation 25 | 26 | The complete documentation of PyUNICORE can be viewed 27 | [here](https://pyunicore.readthedocs.io/en/latest/) 28 | 29 | ## Installation 30 | 31 | Install from PyPI with 32 | 33 | pip install -U pyunicore 34 | 35 | Additional extra packages may be required for your use case: 36 | 37 | * Using the UFTP fuse driver requires "fusepy" 38 | * Using UFTP with pyfilesystem requires "fs" 39 | * Creating JWT tokens signed with keys requires the 40 | "cryptography" package 41 | 42 | You can install (one or more) extras with pip: 43 | 44 | pip install -U pyunicore[crypto,fs,fuse] 45 | 46 | ## Basic usage 47 | 48 | ### Creating a client for a UNICORE site 49 | 50 | ```Python 51 | import pyunicore.client as uc_client 52 | import pyunicore.credentials as uc_credentials 53 | import json 54 | 55 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 56 | 57 | # authenticate with username/password 58 | credential = uc_credentials.UsernamePassword("demouser", "test123") 59 | 60 | client = uc_client.Client(credential, base_url) 61 | print(json.dumps(client.properties, indent = 2)) 62 | ``` 63 | 64 | PyUNICORE supports a variety of 65 | [authentication options](https://pyunicore.readthedocs.io/en/latest/authentication.html). 66 | 67 | ### Run a job and read result files 68 | 69 | ```Python 70 | my_job = {'Executable': 'date'} 71 | 72 | job = client.new_job(job_description=my_job, inputs=[]) 73 | print(json.dumps(job.properties, indent = 2)) 74 | 75 | job.poll() # wait for job to finish 76 | 77 | work_dir = job.working_dir 78 | print(json.dumps(work_dir.properties, indent = 2)) 79 | 80 | stdout = work_dir.stat("/stdout") 81 | print(json.dumps(stdout.properties, indent = 2)) 82 | content = stdout.raw().read() 83 | print(content) 84 | ``` 85 | 86 | ### Connect to a Registry and list all registered services 87 | 88 | ```Python 89 | registry_url = "https://localhost:8080/REGISTRY/rest/registries/default_registry" 90 | 91 | # authenticate with username/password 92 | credential = uc_credentials.UsernamePassword("demouser", "test123") 93 | 94 | r = uc_client.Registry(credential, registry_url) 95 | print(r.site_urls) 96 | ``` 97 | 98 | ### Further reading 99 | 100 | More examples for using PyUNICORE can be found in the "integration-tests" 101 | folder in the source code repository. 102 | 103 | ## UFTP examples 104 | 105 | ### Using UFTP for PyFilesystem 106 | 107 | You can create a [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) `FS` 108 | object either directly in code, or implicitely via a URL. 109 | 110 | The convenient way is via URL: 111 | 112 | ```Python 113 | from fs import open_fs 114 | fs_url = "uftp://demouser:test123@localhost:9000/rest/auth/TEST:/data" 115 | uftp_fs = open_fs(fs_url) 116 | ``` 117 | 118 | [More...](https://pyunicore.readthedocs.io/en/latest/uftp.html#using-uftp-for-pyfilesystem) 119 | 120 | ### Mounting remote filesystems via UFTP 121 | 122 | PyUNICORE contains a FUSE driver based on [fusepy](https://pypi.org/project/fusepy), 123 | allowing you to mount a remote filesystem via UFTP. Mounting is a two step process, 124 | 125 | * authenticate to an Auth server, giving you the UFTPD host/port and one-time password 126 | * run the FUSE driver 127 | 128 | [More...](https://pyunicore.readthedocs.io/en/latest/uftp.html#mounting-remote-filesystems-via-uftp) 129 | 130 | ## Tunneling / port forwarding 131 | 132 | Opens a local server socket for clients to connect to, where traffic 133 | gets forwarded to a service on a HPC cluster login (or compute) node. 134 | This feature requires UNICORE 9.1.0 or later on the server side. 135 | 136 | You can use this feature in two ways 137 | 138 | * in your own applications via the `pyunicore.client.Job` class. 139 | * you can also open a tunnel from the command line using the 140 | 'pyunicore.forwarder' module 141 | 142 | [More...](https://pyunicore.readthedocs.io/en/latest/port_forwarding.html) 143 | 144 | ## Dask cluster implementation (experimental) 145 | 146 | PyUNICORE provides an implementation of a Dask Cluster, allowing to 147 | run the Dask client on your local host (or in a Jupyter notebook in 148 | the Cloud), and have the Dask scheduler and workers running remotely 149 | on the HPC site. 150 | 151 | [More...](https://pyunicore.readthedocs.io/en/latest/dask.html) 152 | 153 | 154 | ### Convert a CWL job to UNICORE 155 | 156 | PyUNICORE provides a tool to convert a CWL CommanLineTool and input into a 157 | UNICORE job file. Given the following YAML files that describe a 158 | CommandLineTool wrapper for the echo command and an input file: 159 | 160 | ```yaml 161 | # echo.cwl 162 | 163 | cwlVersion: v1.2 164 | 165 | class: CommandLineTool 166 | baseCommand: echo 167 | 168 | inputs: 169 | message: 170 | type: string 171 | inputBinding: 172 | position: 1 173 | 174 | outputs: [] 175 | ``` 176 | 177 | ```yaml 178 | # hello_world.yml 179 | 180 | message: "Hello World" 181 | ``` 182 | 183 | A UNICORE job file can be generated using the following command: 184 | 185 | ```bash 186 | unicore-cwl-runner echo.cwl hello_world.yml > hello_world.u 187 | ``` 188 | 189 | ## Helpers 190 | 191 | The `pyunicore.helpers` module provides helper code for: 192 | 193 | * Connecting to 194 | * a Registry (`pyunicore.helpers.connect_to_registry`). 195 | * a site via a Registry URL (`pyunicore.helpers.connect_to_site_from_registry`). 196 | * a site via its core URL (`pyunicore.helpers.connect_to_site`). 197 | * Defining descriptions as a dataclass and easily converting to a `dict` as required by `pyunicore.client.Client.new_job` via a `to_dict()` method: 198 | * `pyunicore.helpers.jobs.Description` for `pyunicore.client.Client.new_job()` 199 | * `pyunicore.helpers.workflows.Description` for `pyunicore.client.WorkflowService.new_workflow()` 200 | * Defining a workflow description 201 | 202 | ### Connecting to a Registry 203 | 204 | ```Python 205 | import json 206 | import pyunicore.credentials as uc_credentials 207 | import pyunicore.helpers as helpers 208 | 209 | registry_url = "https://localhost:8080/REGISTRY/rest/registries/default_registry" 210 | 211 | credentials = uc_credentials.UsernamePassword("demouser", "test123") 212 | 213 | client = helpers.connection.connect_to_registry( 214 | registry_url=registry_url, 215 | credentials=credentials, 216 | ) 217 | print(json.dumps(client.properties, indent=2)) 218 | ``` 219 | 220 | ### Connecting to a site via a Registry 221 | 222 | ```Python 223 | import json 224 | import pyunicore.credentials as uc_credentials 225 | import pyunicore.helpers as helpers 226 | 227 | registry_url = "https://localhost:8080/REGISTRY/rest/registries/default_registry" 228 | site = "DEMO-SITE" 229 | 230 | credentials = uc_credentials.UsernamePassword("demouser", "test123") 231 | 232 | client = helpers.connection.connect_to_site_from_registry( 233 | registry_url=registry_url, 234 | site_name=site, 235 | credentials=credentials, 236 | ) 237 | print(json.dumps(client.properties, indent=2)) 238 | ``` 239 | 240 | ### Connecting to a site directly 241 | 242 | ```Python 243 | import json 244 | import pyunicore.credentials as uc_credentials 245 | import pyunicore.helpers as helpers 246 | 247 | site_url = "https://localhost:8080/DEMO-SITE/rest/core" 248 | 249 | credentials = uc_credentials.UsernamePassword("demouser", "test123") 250 | 251 | client = helpers.connection.connect_to_site( 252 | site_api_url=site_url , 253 | credentials=credentials, 254 | ) 255 | print(json.dumps(client.properties, indent=2)) 256 | ``` 257 | 258 | ### Defining a job or workflow 259 | 260 | ```Python 261 | from pyunicore import helpers 262 | 263 | client = ... 264 | 265 | resources = helpers.jobs.Resources(nodes=4) 266 | job = helpers.jobs.Description( 267 | executable="ls", 268 | project="demoproject", 269 | resources=resources 270 | ) 271 | 272 | client.new_job(job.to_dict()) 273 | ``` 274 | 275 | This works analogously for `pyunicore.helpers.workflows`. 276 | 277 | ## Contributing 278 | 279 | 1. Fork the repository 280 | 2. Install the development dependencies 281 | 282 | ```bash 283 | pip install -r requirements-dev.txt 284 | ``` 285 | 286 | 3. Install pre-commit hooks 287 | 288 | ```bash 289 | pre-commit install 290 | ``` 291 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | 22 | clean: 23 | @find -name "*~" -delete 24 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.https://www.sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=4.3.2 2 | m2r2 3 | sphinx_rtd_theme>=1.0.0 4 | sphinxemoji 5 | docutils>=0.17.1 6 | -------------------------------------------------------------------------------- /docs/source/_static/logo-unicore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HumanBrainProject/pyunicore/bbde277261c3a25757df0942a5aa41bf3e49d933/docs/source/_static/logo-unicore.png -------------------------------------------------------------------------------- /docs/source/authentication.rst: -------------------------------------------------------------------------------- 1 | Authentication 2 | -------------- 3 | 4 | PyUNICORE supports all the authentication options available for 5 | UNICORE, so you can use the correct one for the server that you 6 | are trying to access. 7 | 8 | Basic authentication options 9 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 10 | 11 | The classes for the supported authentication options 12 | are in the `pyunicore.credentials` package. 13 | 14 | 15 | Username and password 16 | ^^^^^^^^^^^^^^^^^^^^^ 17 | 18 | .. code:: python 19 | 20 | import pyunicore.credentials as uc_credentials 21 | 22 | # authenticate with username/password 23 | credential = uc_credentials.UsernamePassword("demouser", "test123") 24 | 25 | This will encode the supplied username/password and add it as an 26 | HTTP header ``Authorization: Basic ...`` to outgoing calls. 27 | 28 | 29 | Bearer token (OAuth/OIDC) 30 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 31 | 32 | This will add the supplied token as an HTTP header 33 | ``Authorization: Bearer ...`` to outgoing calls. 34 | 35 | .. code:: python 36 | 37 | import pyunicore.credentials as uc_credentials 38 | 39 | # authenticate with Bearer token 40 | token = "..." 41 | credential = uc_credentials.OIDCToken(token) 42 | 43 | Basic token 44 | ^^^^^^^^^^^ 45 | 46 | This will add the supplied value as a HTTP header 47 | ``Authorization: Basic ...`` to outgoing calls. 48 | 49 | .. code:: python 50 | 51 | import pyunicore.credentials as uc_credentials 52 | 53 | # authenticate with Bearer token 54 | token = "..." 55 | credential = uc_credentials.BasicToken(token) 56 | 57 | JWT Token 58 | ^^^^^^^^^ 59 | 60 | This is a more complex option that creates a JWT token that is signed 61 | with a private key - for example this is usually an authentication option 62 | supported by the UFTP Authserver. In this case the user's UFTP / SSH key is 63 | used to sign. 64 | 65 | The simplest way to create this credential is to use the 66 | `create_credential()` helper function. 67 | 68 | .. code:: python 69 | 70 | import pyunicore.credentials as uc_credentials 71 | 72 | # authenticate with SSH key 73 | uftp_user = "demouser" 74 | identity_file = "~/.uftp/id_uftp" 75 | credential = uc_credentials.create_credential( 76 | username = uftp_user, 77 | identity = identity_file) 78 | 79 | 80 | The ``JWTToken`` credential can also be used for "trusted services", 81 | where a service uses its server certificate to sign the token. Of 82 | course this must be enabled / supported by the UNICORE server. 83 | 84 | Anonymous access 85 | ^^^^^^^^^^^^^^^^ 86 | 87 | If for some reason you explicitly want anonymous calls, i.e. NO authentication 88 | (which is treated differently from having invalid credentials!), 89 | you can use the ``Anonymous`` credential class: 90 | 91 | .. code:: python 92 | 93 | import pyunicore.credentials as uc_credentials 94 | 95 | # NO authentication 96 | credential = uc_credentials.Anonymous() 97 | 98 | This can be useful for simple health checks and the like. 99 | 100 | User preferences (advanced feature) 101 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 102 | 103 | If the user mapping at the UNICORE server gives you access to more than 104 | one remote user ID or primary group, you can select one using the 105 | `user preferences `_ 106 | feature of the UNICORE REST API. 107 | 108 | The `access_info()` method shows the result of authentication 109 | and authorization. 110 | 111 | .. code:: python 112 | 113 | import json 114 | import pyunicore.client as uc_client 115 | import pyunicore.credentials as uc_credentials 116 | 117 | credential = uc_credentials.UsernamePassword("demouser", "test123") 118 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 119 | client = uc_client.Client(credential, base_url) 120 | 121 | print(json.dumps(client.access_info(), indent=2) 122 | 123 | You can get access to the user preferences via the ``Transport`` object that every 124 | PyUNICORE resource has. 125 | 126 | For example, to select a primary group (from the ones that are available) 127 | 128 | .. code:: python 129 | 130 | client = uc_client.Client(credential, base_url) 131 | client.transport.preferences = "group:myproject1" 132 | 133 | Note that (of course) you cannot select a UID/group that is not available, trying that 134 | will cause a 403 error. 135 | 136 | 137 | Creating an authentication token (advanced feature) 138 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 139 | 140 | For some use cases (like automated workflows) you might want to not store your actual 141 | credentials (like passwords or private keys) for security reasons. For this purpose, you 142 | can use your (secret) credentials to have the UNICORE server issue a (long-lived) 143 | authentication token, that you can then use for your automation tasks without worrying 144 | that your secret credentials get compromised. 145 | 146 | Note that you still should keep this token as secure as possible, since it would allow 147 | anybody who has the token to authenticate to UNICORE with the same permissions and 148 | authorization level as your real credentials. 149 | 150 | You can access the 151 | `token issue endpoint `_ 152 | using the PyUNICORE client class: 153 | 154 | .. code:: python 155 | 156 | client = uc_client.Client(credential, base_url) 157 | my_auth_token = client.issue_auth_token(lifetime = 3600, 158 | renewable = False, 159 | limited = True) 160 | 161 | and later use this token for authentication: 162 | 163 | .. code:: python 164 | 165 | import pyunicore.credentials as uc_credentials 166 | 167 | credential = uc_credential.create_token(token=my_auth_token) 168 | client = uc_client.Client(credential, base_url) 169 | 170 | The parameters are 171 | * ``lifetime`` : token lifetime in seconds 172 | * ``renewable``: if True, the token can be used to issue a new token 173 | * ``limited`` : if True, the token is only valid for the server that issued it. 174 | If False, the token is valid for all UNICORE servers that the 175 | issuing server trusts, i.e. usually those that are in the same UNICORE Registry 176 | -------------------------------------------------------------------------------- /docs/source/basic_usage.rst: -------------------------------------------------------------------------------- 1 | Getting started 2 | --------------- 3 | 4 | Creating a client for a UNICORE site 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | .. code:: python 8 | 9 | import pyunicore.client as uc_client 10 | import pyunicore.credentials as uc_credentials 11 | import json 12 | 13 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 14 | 15 | # authenticate with username/password 16 | credential = uc_credentials.UsernamePassword("demouser", "test123") 17 | 18 | client = uc_client.Client(credential, base_url) 19 | print(json.dumps(client.properties, indent = 2)) 20 | 21 | 22 | Running a job and read result files 23 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 24 | 25 | .. code:: python 26 | 27 | my_job = {'Executable': 'date'} 28 | 29 | job = client.new_job(job_description=my_job, inputs=[]) 30 | print(json.dumps(job.properties, indent = 2)) 31 | 32 | job.poll() # wait for job to finish 33 | 34 | work_dir = job.working_dir 35 | print(json.dumps(work_dir.properties, indent = 2)) 36 | 37 | stdout = work_dir.stat("/stdout") 38 | print(json.dumps(stdout.properties, indent = 2)) 39 | content = stdout.raw().read() 40 | print(content) 41 | 42 | 43 | 44 | Connect to a Registry and list all registered services 45 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 46 | 47 | .. code:: python 48 | 49 | registry_url = "https://localhost:8080/REGISTRY/rest/registries/default_registry" 50 | 51 | # authenticate with username/password 52 | credential = uc_credentials.UsernamePassword("demouser", "test123") 53 | 54 | r = uc_client.Registry(credential, registry_url) 55 | print(r.site_urls) 56 | 57 | 58 | More examples 59 | ~~~~~~~~~~~~~ 60 | 61 | Further examples for using PyUNICORE can be found in the "integration-tests" 62 | folder in the source code repository. 63 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | # -- Path setup -------------------------------------------------------------- 7 | # If extensions (or modules to document with autodoc) are in another directory, 8 | # add these directories to sys.path here. If the directory is relative to the 9 | # documentation root, use os.path.abspath to make it absolute, like shown here. 10 | # 11 | import os 12 | import sys 13 | 14 | sys.path.insert(0, os.path.abspath(".")) 15 | 16 | 17 | def setup(app): 18 | app.add_css_file("css/custom.css") 19 | 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = "PyUNICORE" 24 | author = "2023 UNICORE" 25 | copyright = author 26 | version = "stable" 27 | language = "en" 28 | 29 | 30 | # -- General configuration --------------------------------------------------- 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = [ 36 | "sphinx_rtd_theme", 37 | "sphinx.ext.duration", 38 | "sphinx.ext.doctest", 39 | "sphinx.ext.autodoc", 40 | "sphinx.ext.autosummary", 41 | "sphinx.ext.intersphinx", 42 | "sphinx.ext.todo", 43 | "sphinx.ext.coverage", 44 | "sphinx.ext.ifconfig", 45 | "sphinxemoji.sphinxemoji", 46 | "m2r2", 47 | ] 48 | 49 | # Make sure the target is unique 50 | autosectionlabel_prefix_document = True 51 | 52 | source_suffix = { 53 | ".rst": "restructuredtext", 54 | ".md": "markdown", 55 | } 56 | 57 | # Add any paths that contain templates here, relative to this directory. 58 | templates_path = ["_templates"] 59 | 60 | # List of patterns, relative to source directory, that match files and 61 | # directories to ignore when looking for source files. 62 | # This pattern also affects html_static_path and html_extra_path. 63 | exclude_patterns = ["*.bak", "*.txt", "*.rest"] 64 | 65 | 66 | # -- Options for HTML output ------------------------------------------------- 67 | 68 | # The theme to use for HTML and HTML Help pages. See the documentation for 69 | # a list of builtin themes. 70 | # 71 | html_theme = "sphinx_rtd_theme" # read the docs (external) 72 | html_theme_options = { 73 | "navigation_with_keys": True, 74 | } 75 | html_theme_path = [ 76 | "_themes", 77 | ] 78 | html_theme_options = { 79 | "prev_next_buttons_location": "both", 80 | "collapse_navigation": False, 81 | } 82 | html_logo = "_static/logo-unicore.png" 83 | html_title = "PyUNICORE" 84 | 85 | numfig = True 86 | html_show_sourcelink = False 87 | html_show_sphinx = False 88 | 89 | 90 | # Add any paths that contain custom static files (such as style sheets) here, 91 | # relative to this directory. They are copied after the builtin static files, 92 | # so a file named "default.css" will overwrite the builtin "default.css". 93 | html_static_path = ["_static"] 94 | 95 | 96 | # Tells the project to use sphinx pygments for color coding code examples. 97 | pygments_style = "sphinx" 98 | 99 | html_context = { 100 | "display_github": True, # Add 'Edit on Github' link instead of 'View page source' 101 | } 102 | 103 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 104 | # using the given strftime format. 105 | html_last_updated_fmt = "%b %d, %Y" 106 | -------------------------------------------------------------------------------- /docs/source/dask.rst: -------------------------------------------------------------------------------- 1 | Dask integration 2 | ---------------- 3 | 4 | PyUNICORE provides the ``UNICORECluster`` class, which is an implementation 5 | of a Dask Cluster, allowing to run the Dask client on your local host (or in 6 | a Jupyter notebook in the Cloud), and have the Dask scheduler and workers 7 | running remotely on the HPC site. 8 | 9 | Here is a basic usage example: 10 | 11 | .. code:: python 12 | 13 | import pyunicore.client as uc_client 14 | import pyunicore.credentials as uc_credentials 15 | import pyunicore.dask as uc_dask 16 | 17 | # Create a UNICORE client for accessing the HPC cluster 18 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 19 | credential = uc_credentials.UsernamePassword("demouser", "test123") 20 | submitter = uc_client.Client(credential, base_url) 21 | 22 | # Create the UNICORECluster instance 23 | 24 | uc_cluster = uc_dask.UNICORECluster( 25 | submitter, 26 | queue = "batch", 27 | project = "my-project", 28 | debug=True) 29 | 30 | # Start two workers 31 | uc_cluster.scale(2, wait_for_startup=True) 32 | 33 | # Create a Dask client connected to the UNICORECluster 34 | 35 | from dask.distributed import Client 36 | dask_client = Client(uc_cluster, timeout=120) 37 | 38 | 39 | That's it! Now Dask will run its computations using the scheduler 40 | and workers started via UNICORE on the HPC site. 41 | 42 | 43 | Configuration 44 | ~~~~~~~~~~~~~ 45 | 46 | When creating the ``UNICORECluster``, a number of parameters can be set via the constructor. 47 | All parameters except for the submitter to be used are OPTIONAL. 48 | 49 | - `submitter`: this is either a Client object or an Allocation, which is used to submit new jobs 50 | - `n_workers`: initial number of workers to launch 51 | - `queue`: the batch queue to use 52 | - `project`: the accounting project 53 | - `threads`: worker option controlling the number of threads per worker 54 | - `processes`: worker option controlling the number of worker processes per job (default: 1) 55 | - `scheduler_job_desc`: base job description for launching the scheduler (default: None) 56 | - `worker_job_desc`: base job description for launching a worker (default: None) 57 | - `local_port`: which local port to use for the Dask client (default: 4322) 58 | - `connect_dashboard`: if True, a second forwarding process will be lauched to allow a connection to the dashboard 59 | (default: False) 60 | - `local_dashboard_port`: which local port to use for the dashboard (default: 4323) 61 | - `debug`: if True, print some debug info (default: False) 62 | - `connection_timeout`: timeout in seconds while setting up the port forwarding (default: 120) 63 | 64 | 65 | Customizing the scheduler and workers 66 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 67 | 68 | By default, the Dask extension will launch the Dask components using server-side applications 69 | called ``dask-scheduler`` and ``dask-worker``, which need to be defined in the UNICORE IDB. 70 | 71 | The job description will look like this: 72 | 73 | .. code:: json 74 | 75 | { 76 | "ApplicationName": "dask-scheduler", 77 | "Arguments": [ 78 | "--port", "0", 79 | "--scheduler-file", "./dask.json" 80 | ], 81 | "Resources": { 82 | "Queue": "your_queue", 83 | "Project": "your_project" 84 | } 85 | } 86 | 87 | If you want to customize this, you can pass in a basic job description when creating 88 | the ``UNICORECluster`` object. 89 | 90 | The job descriptions need not contain all command-line arguments, the ``UNICORECluster`` 91 | will add them as required. Also, the queue and project will be set if necessary. 92 | 93 | 94 | For example 95 | 96 | .. code:: python 97 | 98 | # Custom job to start scheduler 99 | 100 | sched_jd = { 101 | "Executable" : "conda run -n dask dask-scheduler", 102 | "Resources": { 103 | "Runtime": "2h" 104 | }, 105 | "Tags": ["dask", "testing"] 106 | } 107 | 108 | # Custom job to start worker 109 | 110 | worker_jd = { 111 | "Executable" : "srun --tasks=1 conda run -n dask dask-worker", 112 | "Resources": { 113 | "Nodes": "2" 114 | } 115 | } 116 | 117 | # Create the UNICORECluster instance 118 | uc_cluster = uc_dask.UNICORECluster( 119 | submitter, 120 | queue = "batch", 121 | project = "my-project", 122 | scheduler_job_desc=sched_jd, 123 | worker_job_desc=worker_jd 124 | ) 125 | 126 | 127 | Scaling 128 | ~~~~~~~ 129 | 130 | To control the number of worker processes and threads, the UNICORECluster has the scale() method, 131 | as well as two properties that can be set from the constructor, or later at runtime 132 | 133 | The scale() method controls how many workers (or worker jobs when using "jobs=..." as argument) 134 | are running. 135 | 136 | .. code:: python 137 | 138 | # Start two workers 139 | uc_cluster.scale(2, wait_for_startup=True) 140 | 141 | # Or start two worker jobs with 4 workers per job 142 | # and 128 threads per worker 143 | uc_cluster.processes = 4 144 | uc_cluster.threads = 128 145 | uc_cluster.scale(jobs=2) 146 | 147 | The dashboard 148 | ~~~~~~~~~~~~~ 149 | 150 | By default a connection to the scheduler's dashboard is not possible. To allow connecting to 151 | the dashboard, set ``connect_dashboard=True`` when creating the ``UNICORECluster``. 152 | The dashboard will then be available at ``http://localhost:4323``, the port can be changed, 153 | if necessary. 154 | 155 | 156 | Using an allocation 157 | ~~~~~~~~~~~~~~~~~~~ 158 | 159 | To speed up the startup and scaling process, it is possible to pre-allocate a multinode batch job 160 | (if the server side UNICORE supports this, i.e. runs UNICORE 9.1 and Slurm), and run the Dask 161 | components in this allocation. 162 | 163 | .. code:: python 164 | 165 | import pyunicore.client as uc_client 166 | import pyunicore.credentials as uc_credentials 167 | import pyunicore.dask as uc_dask 168 | 169 | # Create a UNICORE client for accessing the HPC cluster 170 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 171 | credential = uc_credentials.UsernamePassword("demouser", "test123") 172 | submitter = uc_client.Client(credential, base_url) 173 | 174 | # Allocate a 4-node job 175 | allocation_jd = { 176 | "Job type": "ALLOCATE", 177 | 178 | "Resources": { 179 | "Runtime": "60m", 180 | "Queue": "batch", 181 | "Project": "myproject" 182 | } 183 | } 184 | 185 | allocation = submitter.new_job(allocation_jd) 186 | allocation.wait_until_available() 187 | 188 | # Create the UNICORECluster instance using the allocation 189 | 190 | uc_cluster = uc_dask.UNICORECluster(allocation, debug=True) 191 | 192 | 193 | Note that in this case your custom scheduler / worker job descriptions MUST use ``srun --tasks=1 ...`` 194 | to make sure that exactly one scheduler / worker is started on one node. 195 | 196 | Also make sure to not lauch more jobs than you have nodes - otherwise the new jobs will stay "QUEUED". 197 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to PyUNICORE 2 | ==================== 3 | 4 | `UNICORE `_ (**UN**\ iform **I**\ nterface to 5 | **CO**\ mputing **RE**\ sources) offers a ready-to-run system 6 | including client and server software. It makes distributed computing 7 | and data resources available in a seamless and secure way in intranets 8 | and the internet. 9 | 10 | PyUNICORE is a Python library providing an API for UNICORE's 11 | `REST API `_ , 12 | making common tasks like file access, job submission and management, 13 | workflow submission and management more convenient, and integrating 14 | UNICORE features better with typical Python usage. 15 | 16 | In addition, this library contains code for using 17 | `UFTP `_ (UNICORE FTP) 18 | for filesystem mounts with FUSE, a UFTP driver for 19 | `PyFilesystem `_ 20 | and a UNICORE implementation of a 21 | `Dask Cluster `_ 22 | 23 | This project has received funding from the European Union’s 24 | Horizon 2020 Framework Programme for Research and Innovation under the 25 | Specific Grant Agreement Nos. 720270, 785907 and 945539 26 | (Human Brain Project SGA 1, 2 and 3) 27 | 28 | PyUNICORE is Open Source under the :ref:`BSD License `, 29 | the source code is on `GitHub `_. 30 | 31 | 32 | Installation 33 | ------------ 34 | 35 | Install from PyPI with 36 | 37 | .. code:: console 38 | 39 | pip install -U pyunicore 40 | 41 | Additional extra packages may be required for your use case: 42 | 43 | * Using the UFTP fuse driver requires "fusepy" 44 | * Using UFTP with pyfilesystem requires "fs" 45 | * Creating JWT tokens signed with keys requires the "cryptography" package 46 | 47 | 48 | You can install (one or more) extras with pip: 49 | 50 | .. code:: console 51 | 52 | pip install -U pyunicore[crypto,fs,fuse] 53 | 54 | 55 | .. toctree:: 56 | :maxdepth: 2 57 | :caption: Using PyUNICORE 58 | 59 | basic_usage 60 | authentication 61 | uftp 62 | dask 63 | port_forwarding 64 | 65 | 66 | .. toctree:: 67 | :maxdepth: 1 68 | :caption: Links 69 | 70 | license 71 | -------------------------------------------------------------------------------- /docs/source/license.rst: -------------------------------------------------------------------------------- 1 | .. _license: 2 | 3 | License 4 | ******* 5 | 6 | .. literalinclude:: ../../LICENSE 7 | -------------------------------------------------------------------------------- /docs/source/port_forwarding.rst: -------------------------------------------------------------------------------- 1 | Port forwarding / tunneling 2 | --------------------------- 3 | 4 | Opens a local server socket for clients to connect to, where traffic 5 | gets forwarded to a service on a HPC cluster login (or compute) node. 6 | This feature requires UNICORE 9.1.0 or later on the server side. 7 | 8 | You can use this feature in two ways 9 | 10 | * in your own applications via the ``pyunicore.client.Job`` class. 11 | * you can also open a tunnel from the command line using the ``pyunicore.forwarder`` module 12 | 13 | Here is an example for a command line tool invocation: 14 | 15 | .. code:: console 16 | 17 | LOCAL_PORT=4322 18 | JOB_URL=https://localhost:8080/DEMO-SITE/rest/core/jobs/some_job_id 19 | REMOTE_PORT=8000 20 | python3 -m pyunicore.forwarder --token \ 21 | -L $LOCAL_PORT \ 22 | $JOB_URL/forward-port?port=$REMOTE_PORT \ 23 | 24 | 25 | Your application can now connect to ``localhost:4322`` but all traffic 26 | will be forwarded to port 8000 on the HPC login node where your application 27 | is running. 28 | 29 | See 30 | 31 | .. code:: console 32 | 33 | python3 -m pyunicore.forwarder --help 34 | 35 | for all options. 36 | 37 | If you want to tunnel to a compute node, you need to specify the compute node in your command line: 38 | 39 | .. code:: console 40 | 41 | LOCAL_PORT=4322 42 | JOB_URL=https://localhost:8080/DEMO-SITE/rest/core/jobs/some_job_id 43 | REMOTE_PORT=8000 44 | COMPUTE_NODE=cnode1234 45 | python3 -m pyunicore.forwarder --token \ 46 | -L $LOCAL_PORT \ 47 | $JOB_URL/forward-port?port=$REMOTE_PORT?host=$COMPUTE_NODE \ 48 | -------------------------------------------------------------------------------- /docs/source/uftp.rst: -------------------------------------------------------------------------------- 1 | UFTP 2 | ---- 3 | 4 | `UFTP (UNICORE FTP) `_ is a fast file transfer toolkit, 5 | based on the standard FTP protocol, with an added authentication layer based on UNICORE. 6 | 7 | To make a UFTP connection, a user first needs to authenticate to an 8 | authentication service, which will produce a one-time password, which is 9 | then used to connect to the actual UFTP file server. 10 | 11 | UFTP support in PyUNICORE is based on the `ftplib `_ 12 | standard library. 13 | 14 | Basic UFTP usage 15 | ~~~~~~~~~~~~~~~~ 16 | 17 | Opening an FTP session involves authenticating to an authentication service using 18 | UNICORE credentials. Depending on the authentication service, different credentials 19 | might be accepted. 20 | 21 | Here is a basic example using username/password. 22 | 23 | .. code:: python 24 | 25 | import pyunicore.credentials as uc_credentials 26 | import pyunicore.uftp as uc_uftp 27 | 28 | # URL of the authentication service 29 | auth_url = "https://localhost:9000/rest/auth/TEST" 30 | 31 | # remote base directory that we want to access 32 | base_directory = "/data" 33 | 34 | # authenticate with username/password 35 | credential = uc_credentials.UsernamePassword("demouser", "test123") 36 | 37 | uftp_session = uc_uftp.UFTP().connect(credential, auth_url, base_directory) 38 | 39 | The object returned by `connect()` is an `ftplib` `FTP` object. 40 | 41 | 42 | Mounting remote filesystems via UFTP and FUSE 43 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 44 | 45 | PyUNICORE contains a FUSE driver based on `fusepy `_, 46 | allowing you to mount a remote filesystem via UFTP. Mounting is a two step process, 47 | 48 | * authenticate to an Auth server, giving you the UFTPD host/port and one-time password 49 | * run the FUSE driver 50 | 51 | The following code example gives you the basic idea: 52 | 53 | .. code:: python 54 | 55 | import pyunicore.client as uc_client 56 | import pyunicore.credentials as uc_credentials 57 | import pyunicore.uftp.uftp as uc_uftp 58 | import pyunicore.uftp.uftpfuse as uc_fuse 59 | 60 | _auth = "https://localhost:9000/rest/auth/TEST" 61 | _base_dir = "/opt/shared-data" 62 | _local_mount_dir = "/tmp/mount" 63 | 64 | # authenticate 65 | cred = uc_credentials.UsernamePassword("demouser", "test123") 66 | _host, _port, _password = uc_uftp.UFTP().authenticate(cred, _auth, _base_dir) 67 | 68 | # run the fuse driver 69 | fuse = uc_fuse.FUSE( 70 | uc_fuse.UFTPDriver(_host, _port, _password), _local_mount_dir, foreground=False, nothreads=True) 71 | 72 | 73 | Using UFTP for PyFilesystem 74 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 75 | 76 | `PyFilesystem `_ is a virtual filesystem 77 | for Python supporting a variety of protocols. 78 | 79 | PyUNICORE contains two "drivers" that leverage UFTP. 80 | 81 | The first one accesses data directly via UFTP. You can create a PyFilesystem `FS` 82 | object either directly in code, or implicitely via a URL. 83 | The convenient way is via URL: 84 | 85 | .. code:: python 86 | 87 | from fs import open_fs 88 | fs_url = "uftp://demouser:test123@localhost:9000/rest/auth/TEST:/data" 89 | uftp_fs = open_fs(fs_url) 90 | 91 | 92 | The URL format is 93 | 94 | .. code:: console 95 | 96 | uftp://[username]:[password]@[auth-server-url]:[base-directory]?[token=...][identity=...] 97 | 98 | 99 | The FS driver supports three types of authentication 100 | 101 | * Username/Password - give `username` and `password` 102 | * SSH Key - give `username` and the `identity` parameter, 103 | where `identity` is the filename of a private key. 104 | Specify the `password` if needed to load the private key 105 | * Bearer token - give the token value via the `token` parameter 106 | 107 | The `base-directory` parameter denotes the remote directory that is to be accessed. 108 | 109 | The second driver mounts the remote filesystem via FUSE, and then accesses 110 | data "locally". The URL format is 111 | 112 | .. code:: console 113 | 114 | uftpmount://[username]:[password]@[auth-server-url]:[base-directory==mount-directory]?[token=...][identity=...] 115 | 116 | with the same authentication options as before. 117 | 118 | The mount directory is given 119 | 120 | The `base-directory==mount-directory` parameter denotes the remote directory that is to be accessed, 121 | as well as the local directory where it should be mounted. 122 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.pytest.ini_options] 2 | addopts = "--cov=pyunicore --cov-report term-missing" 3 | 4 | [tool.black] 5 | line-length = 100 6 | 7 | [tool.flake8] 8 | max-line-length = 100 9 | ignore = [ 10 | "W503" 11 | ] 12 | per-file-ignores = [ 13 | "__init__.py:F401,E501", 14 | ] 15 | exclude = [ 16 | "versioneer.py", 17 | "pyunicore/_version.py", 18 | "setup.py", 19 | ] 20 | -------------------------------------------------------------------------------- /pyunicore/__init__.py: -------------------------------------------------------------------------------- 1 | from . import _version 2 | 3 | __version__ = _version.get_versions()["version"] 4 | -------------------------------------------------------------------------------- /pyunicore/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HumanBrainProject/pyunicore/bbde277261c3a25757df0942a5aa41bf3e49d933/pyunicore/cli/__init__.py -------------------------------------------------------------------------------- /pyunicore/cli/info.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | 5 | from pyunicore.cli.base import Base 6 | from pyunicore.client import Resource 7 | 8 | 9 | class Info(Base): 10 | def add_command_args(self): 11 | self.parser.prog = "unicore system-info" 12 | self.parser.description = self.get_synopsis() 13 | self.parser.add_argument("URL", help="Endpoint URL(s)", nargs="*") 14 | self.parser.add_argument( 15 | "-p", 16 | "--pattern", 17 | required=False, 18 | type=str, 19 | help="Only show info for endpoints matching the given regexp", 20 | ) 21 | self.parser.add_argument( 22 | "-l", "--long", required=False, action="store_true", help="Show detailed info" 23 | ) 24 | 25 | def _require_registry(self): 26 | return True 27 | 28 | def get_synopsis(self): 29 | return """Show information about endpoint(s). If no explicit endpoints are given, 30 | the endpoints in the registry are used. The optional pattern allows to limit which 31 | endpoints are listed.""" 32 | 33 | def get_description(self): 34 | return "show info on available services" 35 | 36 | def get_group(self): 37 | return "Utilities" 38 | 39 | def run(self, args): 40 | super().setup(args) 41 | endpoints = self.registry.site_urls.values() 42 | 43 | if self.args.URL: 44 | endpoints = self.args.URL 45 | 46 | for url in endpoints: 47 | if self.args.pattern: 48 | if not re.match(self.args.pattern, url): 49 | continue 50 | c = Resource(self.credential, resource_url=url) 51 | self.show_endpoint_details(c) 52 | 53 | def show_endpoint_details(self, ep: Resource): 54 | print(ep.resource_url) 55 | if ep.resource_url.endswith("/rest/core"): 56 | self._show_details_core(ep) 57 | elif re.match(".*/rest/core/storages/.+", ep.resource_url): 58 | self._show_details_storage(ep) 59 | else: 60 | print(" * no further details available.") 61 | 62 | def _show_details_core(self, ep: Resource): 63 | props = ep.properties 64 | print(" * type: UNICORE/X base") 65 | print(f" * server v{props['server']['version']}") 66 | xlogin = props["client"]["xlogin"] 67 | role = props["client"]["role"]["selected"] 68 | uid = xlogin.get("UID", "n/a") 69 | print(f" * authenticated as: '{props['client']['dn']}' role='{role}' uid='{uid}'") 70 | grps = xlogin.get("availableGroups", []) 71 | uids = xlogin.get("availableUIDs", []) 72 | if len(uids) > 0: 73 | print(f" * available UIDs: {uids}") 74 | if len(grps) > 0: 75 | print(f" * available groups: {grps}") 76 | roles = props["client"]["role"].get("availableRoles", []) 77 | if len(roles) > 0: 78 | print(f" * available roles: {roles}") 79 | 80 | def _show_details_storage(self, ep: Resource): 81 | props = ep.properties 82 | t = "storage" 83 | if ep.resource_url.endswith("-uspace"): 84 | t = t + " (job directory)" 85 | print(f" * type: {t}") 86 | print(f" * mount point: {props['mountPoint']}") 87 | print(f" * free space : {int(props['freeSpace']/1024/1024)} MB") 88 | -------------------------------------------------------------------------------- /pyunicore/cli/main.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import platform 4 | import sys 5 | 6 | import pyunicore.cli.base 7 | import pyunicore.cli.exec 8 | import pyunicore.cli.info 9 | import pyunicore.cli.io 10 | 11 | _commands = { 12 | "cancel-job": pyunicore.cli.exec.CancelJob, 13 | "cat": pyunicore.cli.io.Cat, 14 | "cp": pyunicore.cli.io.CP, 15 | "exec": pyunicore.cli.exec.Exec, 16 | "info": pyunicore.cli.info.Info, 17 | "issue-token": pyunicore.cli.base.IssueToken, 18 | "job-status": pyunicore.cli.exec.GetJobStatus, 19 | "list-jobs": pyunicore.cli.exec.ListJobs, 20 | "ls": pyunicore.cli.io.LS, 21 | "rest": pyunicore.cli.base.REST, 22 | "run": pyunicore.cli.exec.Run, 23 | } 24 | 25 | 26 | def get_command(name): 27 | return _commands.get(name)() 28 | 29 | 30 | def show_version(): 31 | print( 32 | "UNICORE Commandline Client (pyUNICORE) " 33 | "%s, https://www.unicore.eu" % pyunicore._version.get_versions().get("version", "n/a") 34 | ) 35 | print("Python %s" % sys.version) 36 | print("OS: %s" % platform.platform()) 37 | 38 | 39 | def help(): 40 | s = """UNICORE Commandline Client (pyUNICORE) %s, https://www.unicore.eu 41 | Usage: unicore [OPTIONS] 42 | The following commands are available:""" % pyunicore._version.get_versions().get( 43 | "version", "n/a" 44 | ) 45 | print(_header) 46 | print(s) 47 | for cmd in sorted(_commands): 48 | print(f" {cmd:20} - {get_command(cmd).get_description()}") 49 | print("Enter 'unicore -h' for help on a particular command.") 50 | 51 | 52 | def run(args): 53 | _help = ["help", "-h", "--help"] 54 | if len(args) < 1 or args[0] in _help: 55 | help() 56 | return 57 | _version = ["version", "-V", "--version"] 58 | if args[0] in _version: 59 | show_version() 60 | return 61 | 62 | command = None 63 | cmd = args[0] 64 | for k in _commands: 65 | if k.startswith(cmd): 66 | command = get_command(k) 67 | break 68 | if command is None: 69 | raise ValueError(f"No such command: {cmd}") 70 | command.run(args[1:]) 71 | 72 | 73 | _header = """ _ _ _ _ _____ _____ ____ _____ ______ 74 | | | | | \\ | |_ _/ ____/ __ \\| __ \\| ____| 75 | | | | | \\| | | || | | | | | |__) | |__ 76 | | | | | . ` | | || | | | | | _ /| __| 77 | | |__| | |\\ |_| |_ |____ |__| | | \\ \\| |____ 78 | \\____/|_| \\_|_____\\_____\\____/|_| \\_\\______| 79 | """ 80 | 81 | 82 | def main(): 83 | """ 84 | Main entry point 85 | """ 86 | run(sys.argv[1:]) 87 | 88 | 89 | if __name__ == "__main__": 90 | main() 91 | -------------------------------------------------------------------------------- /pyunicore/cwl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HumanBrainProject/pyunicore/bbde277261c3a25757df0942a5aa41bf3e49d933/pyunicore/cwl/__init__.py -------------------------------------------------------------------------------- /pyunicore/cwl/cwlconverter.py: -------------------------------------------------------------------------------- 1 | """ 2 | CWL to UNICORE converter and utilities 3 | """ 4 | 5 | 6 | def convert_cmdline_tool(cwl_doc, inputs_object={}, debug=False): 7 | """converts a CWL CommandLineTool into a UNICORE JSON job 8 | 9 | Returns: UNICORE JSON job, list of local files to upload, list of output files 10 | """ 11 | if cwl_doc["class"] != "CommandLineTool": 12 | raise ValueError("Unsupported 'class' of CWL document, must be 'CommandLineTool'") 13 | unicore_job = {} 14 | 15 | _hints = cwl_doc.get("hints", {}) 16 | 17 | _is_container = _hints.get("DockerRequirement") is not None 18 | if debug: 19 | print("Container mode: %s" % _is_container) 20 | 21 | if _is_container: 22 | unicore_job["ApplicationName"] = "CONTAINER" 23 | docker_image = _hints["DockerRequirement"]["dockerPull"] 24 | params = { 25 | "IMAGE_URL": docker_image, 26 | "COMMAND": cwl_doc["baseCommand"], 27 | } 28 | unicore_job["Parameters"] = params 29 | else: 30 | unicore_job["Executable"] = cwl_doc["baseCommand"] 31 | 32 | unicore_job["Arguments"] = build_argument_list(cwl_doc.get("inputs", {}), inputs_object, debug) 33 | 34 | if "stdout" in cwl_doc.keys(): 35 | unicore_job["Stdout"] = cwl_doc["stdout"] 36 | if "stderr" in cwl_doc.keys(): 37 | unicore_job["Stderr"] = cwl_doc["stderr"] 38 | if "stdin" in cwl_doc.keys(): 39 | unicore_job["Stdin"] = cwl_doc["stdin"] 40 | 41 | remote_files = get_remote_file_list(inputs_object) 42 | if len(remote_files) > 0: 43 | unicore_job["Imports"] = remote_files 44 | files = get_local_file_list(inputs_object) 45 | outputs = [] 46 | 47 | return unicore_job, files, outputs 48 | 49 | 50 | def build_argument_list(cwl_inputs, inputs_object={}, debug=False): 51 | """generate the argument list from the CWL inputs and an inputs_object containing values""" 52 | render = {} 53 | for i in cwl_inputs: 54 | input_item = cwl_inputs[i] 55 | input_binding = input_item.get("inputBinding", None) 56 | if input_binding is not None: 57 | pos = int(input_binding["position"]) 58 | value = render_value(i, input_item, inputs_object) 59 | if value is not None: 60 | render[pos] = value 61 | args = [] 62 | for _, value in sorted(render.items(), key=lambda x: x[0]): 63 | for x in value: 64 | args.append(x) 65 | return args 66 | 67 | 68 | def render_value(name, input_spec, inputs_object={}): 69 | """generate a concrete value for command-line argument""" 70 | value = inputs_object.get(name, None) 71 | parameter_type = input_spec["type"] 72 | input_binding = input_spec.get("inputBinding", {}) 73 | 74 | is_array = False 75 | is_nested_array = False 76 | 77 | if isinstance(parameter_type, dict): 78 | is_array = True 79 | is_nested_array = True 80 | input_binding = parameter_type.get("inputBinding", {}) 81 | parameter_type = parameter_type["items"] 82 | elif parameter_type.endswith("[]"): 83 | is_array = True 84 | parameter_type = parameter_type[:-2] 85 | 86 | prefix = input_binding.get("prefix", "") 87 | item_separator = input_binding.get("itemSeparator", None) 88 | separate = prefix != "" and input_binding.get("separate", True) 89 | 90 | if parameter_type.endswith("?"): 91 | parameter_type = parameter_type[:-1] 92 | if value is None: 93 | return None 94 | elif value is None: 95 | raise TypeError("Parameter value for parameter '%s' is missing in inputs object" % name) 96 | 97 | if parameter_type == "boolean": 98 | if (isinstance(value, bool) and value) or value == "true": 99 | return prefix 100 | else: 101 | return None 102 | 103 | if is_array: 104 | if not isinstance(value, list): 105 | raise ValueError( 106 | "Parameter '{}' is declared as 'array of {}', but value is not a list".format( 107 | name, parameter_type 108 | ) 109 | ) 110 | values = value 111 | else: 112 | values = [value] 113 | 114 | resolved_values = [] 115 | 116 | for v in values: 117 | if parameter_type == "string" and " " in v: 118 | current_value = '"' + v + '"' 119 | elif parameter_type == "File" or parameter_type == "Directory": 120 | current_value = get_filename_in_jobdir(v) 121 | else: 122 | current_value = str(v) 123 | resolved_values.append(current_value) 124 | 125 | if item_separator is not None: 126 | resolved_values = [item_separator.join(resolved_values)] 127 | 128 | first = True 129 | result = [] 130 | for v in resolved_values: 131 | if first: 132 | if prefix != "": 133 | if not separate: 134 | result.append(prefix + v) 135 | else: 136 | result.append(prefix) 137 | result.append(v) 138 | else: 139 | result.append(v) 140 | else: 141 | if prefix != "": 142 | if not separate: 143 | result.append(prefix + v) 144 | else: 145 | if is_nested_array: 146 | result.append(prefix) 147 | result.append(v) 148 | else: 149 | result.append(v) 150 | first = False 151 | 152 | return result 153 | 154 | 155 | def get_local_file_list(inputs_object={}): 156 | file_list = [] 157 | for x in inputs_object: 158 | input_item = inputs_object[x] 159 | try: 160 | if input_item.get("class", None) == "File": 161 | path = input_item.get("path", None) 162 | if path is None: 163 | path = input_item.get("location", "") 164 | if not path.startswith("file:"): 165 | continue 166 | path = path[5:] 167 | file_list.append(path) 168 | elif input_item.get("class", None) == "Directory": 169 | # TBD resolve 170 | pass 171 | except AttributeError: 172 | pass 173 | return file_list 174 | 175 | 176 | def get_remote_file_list(inputs_object={}): 177 | file_list = [] 178 | for x in inputs_object: 179 | input_item = inputs_object[x] 180 | try: 181 | if input_item.get("class", None) == "File": 182 | path = input_item.get("path", None) 183 | if path is not None: 184 | continue 185 | path = input_item.get("location") 186 | if path.startswith("file:"): 187 | continue 188 | base_name = input_item.get("basename", None) 189 | if base_name is None: 190 | base_name = path.split("/")[-1] 191 | file_list.append({"From": path, "To": base_name}) 192 | except AttributeError: 193 | pass 194 | return file_list 195 | 196 | 197 | def get_filename_in_jobdir(input_item): 198 | name = input_item.get("path", None) 199 | if name is None: 200 | name = input_item.get("basename", None) 201 | if name is None: 202 | name = input_item.get("location").split("/")[-1] 203 | return name 204 | -------------------------------------------------------------------------------- /pyunicore/cwl/cwltool.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import sys 4 | 5 | import yaml 6 | 7 | import pyunicore.cwl.cwlconverter as cwlconverter 8 | 9 | 10 | def read_cwl_files(cwl_doc_path, cwl_inputs_object_path=None, debug=False): 11 | with open(cwl_doc_path) as f: 12 | if debug: 13 | print("Reading CWL from: %s" % cwl_doc_path) 14 | cwl_doc = yaml.safe_load(f) 15 | cwl_inputs_object = {} 16 | if cwl_inputs_object_path is not None: 17 | if debug: 18 | print("Reading parameter values from: %s" % cwl_inputs_object_path) 19 | with open(cwl_inputs_object_path) as f: 20 | try: 21 | cwl_inputs_object = yaml.safe_load(f) 22 | except yaml.error.YAMLError: 23 | with open(cwl_inputs_object_path) as f2: 24 | cwl_inputs_object = json.load(f2) 25 | return cwl_doc, cwl_inputs_object 26 | 27 | 28 | def main(): 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument( 31 | "cwl_document", 32 | metavar="cwl_document", 33 | help="Path to a CWL CommandLineTool", 34 | ) 35 | parser.add_argument( 36 | "inputs_object", 37 | nargs="?", 38 | metavar="inputs_object", 39 | help="Path to a YAML or JSON " 40 | "formatted description of the required input values for the given `cwl_document`.", 41 | ) 42 | parser.add_argument("-d", "--debug", action="store_true", help="Debug mode") 43 | args = parser.parse_args() 44 | 45 | cwl_doc_path = args.cwl_document 46 | cwl_inputs_object_path = args.inputs_object 47 | debug = args.debug 48 | cwl_doc, cwl_inputs_object = read_cwl_files(cwl_doc_path, cwl_inputs_object_path, debug) 49 | 50 | ( 51 | unicore_job, 52 | file_list, 53 | outputs_list, 54 | ) = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_inputs_object, debug=debug) 55 | print(json.dumps(unicore_job, indent=2, sort_keys=True)) 56 | 57 | sys.exit(0) 58 | 59 | 60 | if __name__ == "__main__": 61 | main() 62 | -------------------------------------------------------------------------------- /pyunicore/forwarder.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import ssl 3 | import threading 4 | from urllib.parse import urlparse 5 | 6 | from pyunicore.client import Transport 7 | from pyunicore.credentials import create_credential 8 | 9 | 10 | class Forwarder: 11 | """Forwarding helper""" 12 | 13 | def __init__( 14 | self, 15 | transport, 16 | endpoint, 17 | service_port=None, 18 | service_host=None, 19 | login_node=None, 20 | debug=False, 21 | ): 22 | """Creates a new Forwarder instance 23 | The remote service host/port can be already encoded in the endpoint, or given separately 24 | 25 | Args: 26 | transport: the transport (security sessions should be OFF) 27 | endpoint: UNICORE REST API endpoint which can establish the forwarding 28 | service_port: the remote service port (if not already encoded in the endpoint) 29 | service_host: the (optional) remote service host (if not encoded in the endpoint) 30 | login_node: the /optional) login node to use (if not encoded in the endpoint) 31 | debug: set to True for some debug output to the console 32 | """ 33 | self.endpoint = endpoint 34 | self.parsed_url = _parse_forwarding_params( 35 | self.endpoint, service_port, service_host, login_node 36 | ) 37 | self.transport = transport 38 | self.quiet = not debug 39 | self.local_port = 0 40 | self.service_socket = None 41 | self.client_socket = None 42 | 43 | def connect(self): 44 | """connect to the backend service and return the connected socket""" 45 | sock = self.create_socket() 46 | msg = [ 47 | f"GET {self.parsed_url.path}?{self.parsed_url.query} HTTP/1.1", 48 | "Host: %s" % self.parsed_url.netloc, 49 | "Connection: Upgrade", 50 | "Upgrade: UNICORE-Socket-Forwarding", 51 | ] 52 | headers = self.transport._headers({}) 53 | for h in headers: 54 | msg.append(f"{h}: {headers[h]}") 55 | msg.append("") 56 | for m in msg: 57 | sock.write(bytes(m + "\r\n", "UTF-8")) 58 | if self.quiet: 59 | continue 60 | if m.startswith("Authorization"): 61 | print("<-- Authorization: ***") 62 | else: 63 | print("<--", m) 64 | reader = sock.makefile("r") 65 | first = True 66 | code = -1 67 | while True: 68 | line = reader.readline().strip() 69 | self.quiet or print("--> %s" % line) 70 | if len(line) == 0: 71 | break 72 | if first: 73 | code = int(line.split(" ")[1]) 74 | first = False 75 | if code != 101: 76 | raise ValueError( 77 | "Backend returned HTTP %s, could not handle UNICORE-Socket-Forwarding" % code 78 | ) 79 | self.quiet or print("Connected to backend service.") 80 | return sock 81 | 82 | def create_socket(self): 83 | self.quiet or print("Connecting to %s" % self.parsed_url.netloc) 84 | if ":" in self.parsed_url.netloc: 85 | _addr = self.parsed_url.netloc.split(":") 86 | address = (_addr[0], int(_addr[1])) 87 | else: 88 | if "https" == self.parsed_url.scheme.lower(): 89 | address = (self.parsed_url.netloc, 443) 90 | else: 91 | address = (self.parsed_url.netloc, 80) 92 | sock = socket.create_connection(address) 93 | if "https" == self.parsed_url.scheme: 94 | context = ssl.SSLContext(ssl.PROTOCOL_TLS) 95 | context.verify_mode = ssl.CERT_NONE 96 | context.check_hostname = False 97 | context.load_default_certs() 98 | sock = context.wrap_socket(sock) 99 | return sock 100 | 101 | def start_forwarding(self): 102 | self.quiet or print("Start forwarding.") 103 | threading.Thread( 104 | target=self.transfer, args=(self.client_socket, self.service_socket) 105 | ).start() 106 | threading.Thread( 107 | target=self.transfer, args=(self.service_socket, self.client_socket) 108 | ).start() 109 | 110 | def stop_forwarding(self): 111 | try: 112 | if self.client_socket: 113 | self.client_socket.close() 114 | except OSError: 115 | pass 116 | try: 117 | if self.service_socket: 118 | self.service_socket.close() 119 | except OSError: 120 | pass 121 | 122 | def transfer(self, source, destination): 123 | desc = f"{source.getpeername()} --> {destination.getpeername()}" 124 | self.quiet or print("Start TCP forwarding %s" % desc) 125 | buf_size = 32768 126 | while True: 127 | try: 128 | buffer = source.recv(buf_size) 129 | if len(buffer) > 0: 130 | destination.send(buffer) 131 | elif len(buffer) == 0: 132 | self.quiet or print("Source is at EOF for %s" % desc) 133 | break 134 | except OSError as e: 135 | self.quiet or print("I/O ERROR for %s " % desc, e) 136 | for s in source, destination: 137 | try: 138 | s.close() 139 | except OSError: 140 | pass 141 | break 142 | self.quiet or print("Stopping TCP forwarding %s" % desc) 143 | 144 | def run(self, local_port): 145 | """open a listener, accept client connections and forward them to the backend""" 146 | with socket.socket() as server: 147 | server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 148 | server.bind(("", local_port)) 149 | if local_port == 0: 150 | print("Listening on %s" % str(server.getsockname())) 151 | self.local_port = server.getsockname()[1] 152 | server.listen(2) 153 | while True: 154 | self.quiet or print("Waiting for client connection.") 155 | self.client_socket, _ = server.accept() 156 | self.quiet or print("Client %s connected." % str(self.client_socket.getpeername())) 157 | self.service_socket = self.connect() 158 | self.start_forwarding() 159 | 160 | 161 | def _parse_forwarding_params(endpoint, service_port=None, service_host=None, login_node=None): 162 | """If not already present in the endpoint, the parameters like 163 | service_port are added. 164 | 165 | Returns: 166 | parsed URL with query parameters added as needed 167 | """ 168 | parsed_url = urlparse(endpoint) 169 | q = parsed_url.query 170 | if service_port is not None and "port=" not in endpoint: 171 | if len(q) > 0: 172 | q += "&" 173 | q += "port=%d" % service_port 174 | if service_host is not None and "host=" not in endpoint: 175 | if len(q) > 0: 176 | q += "&" 177 | q += "host=%s" % service_host 178 | if login_node is not None and "loginNode=" not in endpoint: 179 | if len(q) > 0: 180 | q += "&" 181 | q += "loginNode=%s" % login_node 182 | return parsed_url._replace(query=q) 183 | 184 | 185 | def open_tunnel(job, service_port=None, service_host=None, login_node=None, debug=False): 186 | """open a tunnel to a service running on the HPC side 187 | and return the connected socket 188 | """ 189 | endpoint = job.links["forwarding"] 190 | tr = job.transport._clone() 191 | tr.use_security_sessions = False 192 | forwarder = Forwarder(tr, endpoint, service_port, service_host, login_node, debug) 193 | return forwarder.connect() 194 | 195 | 196 | def run_forwarder(tr, local_port, endpoint, debug): 197 | """Starts a loop listening on 'local_port' for client connections. 198 | It connect clients to the backend 'endpoint' 199 | 200 | Args: 201 | transport: the transport (security sessions should be OFF) 202 | local_port: local port to listen on (use 0 to listen on any free port) 203 | endpoint: UNICORE REST API endpoint which can establish the forwarding 204 | debug: set to True for some debug output to the console 205 | """ 206 | Forwarder(tr, endpoint, debug=debug).run(local_port) 207 | 208 | 209 | def main(): 210 | """ 211 | Main function to listen on a local port for a client connection. 212 | Once the client connects, the tool contacts the server and negotiates 213 | the port forwarding 214 | """ 215 | import argparse 216 | 217 | parser = argparse.ArgumentParser() 218 | parser.add_argument("endpoint", help="Full UNICORE REST API endpoint for forwarding") 219 | parser.add_argument("-L", "--listen", required=True, help="local port to listen on") 220 | parser.add_argument( 221 | "-d", "--debug", required=False, action="store_true", help="print debug info" 222 | ) 223 | parser.add_argument("-t", "--token", help="Authentication: token") 224 | parser.add_argument("-u", "--username", help="Authentication: username") 225 | parser.add_argument( 226 | "-p", 227 | "--password", 228 | nargs="?", 229 | const="__ASK__", 230 | help="Authentication: password (leave empty to enter interactively)", 231 | ) 232 | parser.add_argument("-i", "--identity", help="Authentication: private key file") 233 | args = parser.parse_args() 234 | 235 | port = int(args.listen) 236 | endpoint = args.endpoint 237 | password = args.password 238 | if "__ASK__" == password: 239 | import getpass 240 | 241 | password = getpass.getpass("Enter password:") 242 | credential = create_credential(args.username, password, args.token, args.identity) 243 | tr = Transport(credential, use_security_sessions=False) 244 | run_forwarder(tr, port, endpoint, args.debug) 245 | 246 | 247 | if __name__ == "__main__": 248 | main() 249 | -------------------------------------------------------------------------------- /pyunicore/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers import connection 2 | from pyunicore.helpers import jobs 3 | from pyunicore.helpers import workflows 4 | -------------------------------------------------------------------------------- /pyunicore/helpers/_api_object.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import Any 3 | from typing import Dict 4 | from typing import List 5 | from typing import Union 6 | 7 | 8 | class ApiRequestObject(abc.ABC): 9 | """Any object for API requests.""" 10 | 11 | def to_dict(self) -> Dict: 12 | """Return as dict.""" 13 | as_dict = self._to_dict() 14 | return _create_dict_with_not_none_values(as_dict) 15 | 16 | @abc.abstractmethod 17 | def _to_dict(self) -> Dict: 18 | """Return as dict.""" 19 | 20 | 21 | def _create_dict_with_not_none_values(kwargs: Dict) -> Dict: 22 | return {key: _convert_value(value) for key, value in kwargs.items() if value is not None} 23 | 24 | 25 | def _convert_value(value: Union[Any, ApiRequestObject]) -> Any: 26 | if isinstance(value, dict): 27 | return _create_dict_with_not_none_values(value) 28 | elif isinstance(value, (list, tuple, set)): 29 | return _create_list_with_not_none_values(value) 30 | elif isinstance(value, ApiRequestObject): 31 | return value.to_dict() 32 | elif isinstance(value, bool): 33 | return str(value).lower() 34 | return value 35 | 36 | 37 | def _create_list_with_not_none_values(values: List) -> List: 38 | return [_convert_value(value) for value in values if value is not None] 39 | -------------------------------------------------------------------------------- /pyunicore/helpers/connection/__init__.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.connection.registry import connect_to_registry 2 | from pyunicore.helpers.connection.registry import connect_to_site_from_registry 3 | from pyunicore.helpers.connection.site import connect_to_site 4 | -------------------------------------------------------------------------------- /pyunicore/helpers/connection/registry.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import pyunicore.client 4 | from pyunicore import credentials 5 | from pyunicore.helpers.connection import site as _site 6 | 7 | 8 | def connect_to_registry( 9 | registry_url: str, credential: credentials.Credential 10 | ) -> pyunicore.client.Registry: 11 | """Connect to a registry. 12 | 13 | Args: 14 | registry_url (str): URL to the UNICORE registry. 15 | credential (pyunicore.credentials.Credential): Authentication method. 16 | 17 | Returns: 18 | pyunicore.client.Registry 19 | 20 | """ 21 | return pyunicore.client.Registry(credential, url=registry_url) 22 | 23 | 24 | def connect_to_site_from_registry( 25 | registry_url: str, site_name: str, credential: credentials.Credential 26 | ) -> pyunicore.client.Client: 27 | """Create a connection to a site's UNICORE API from the registry base URL. 28 | 29 | Args: 30 | registry_url (str): URL to the UNICORE registry. 31 | site_name (str): Name of the site to connect to. 32 | credential (pyunicore.credentials.Credential): Authentication method. 33 | 34 | Raises: 35 | ValueError: Site not available in the registry. 36 | 37 | Returns: 38 | pyunicore.client.Client 39 | 40 | """ 41 | site_api_url = _get_site_api_url( 42 | site=site_name, credential=credential, registry_url=registry_url 43 | ) 44 | client = _site.connect_to_site( 45 | site_api_url=site_api_url, 46 | credential=credential, 47 | ) 48 | return client 49 | 50 | 51 | def _get_site_api_url( 52 | site: str, 53 | credential: credentials.Credential, 54 | registry_url: str, 55 | ) -> str: 56 | api_urls = _get_api_urls(credential, registry_url=registry_url) 57 | try: 58 | api_url = api_urls[site] 59 | except KeyError: 60 | available_sites_list = list(api_urls.keys()) 61 | available_sites = ", ".join(available_sites_list) 62 | raise ValueError( 63 | f"Site {site} not available in registry {registry_url}. " 64 | f"Available sites: {available_sites}." 65 | ) 66 | return api_url 67 | 68 | 69 | def _get_api_urls(credential: credentials.Credential, registry_url: str) -> Dict[str, str]: 70 | registry = pyunicore.client.Registry(credential, url=registry_url) 71 | return registry.site_urls 72 | -------------------------------------------------------------------------------- /pyunicore/helpers/connection/site.py: -------------------------------------------------------------------------------- 1 | import pyunicore.client 2 | from pyunicore import credentials as _credentials 3 | 4 | 5 | def connect_to_site( 6 | site_api_url: str, credential: _credentials.Credential 7 | ) -> pyunicore.client.Client: 8 | """Create a connection to a site's UNICORE API. 9 | 10 | Args: 11 | site_api_url (str): REST API URL to the cluster's UNICORE server. 12 | credentials (pyunicore.credentials.Credential): Authentication method. 13 | 14 | Raises: 15 | pyunicore.credentials.AuthenticationFailedException: Authentication on the cluster failed. 16 | 17 | Returns: 18 | pyunicore.client.Client 19 | 20 | """ 21 | client = _connect_to_site( 22 | api_url=site_api_url, 23 | credential=credential, 24 | ) 25 | if _authentication_failed(client): 26 | raise _credentials.AuthenticationFailedException( 27 | "Check if your credentials are correct, and if the cluster name " 28 | "and registry URL are correct." 29 | ) 30 | return client 31 | 32 | 33 | def _connect_to_site(api_url: str, credential: _credentials.Credential) -> pyunicore.client.Client: 34 | client = _create_client(credential=credential, api_url=api_url) 35 | return client 36 | 37 | 38 | def _create_client(credential: _credentials.Credential, api_url: str) -> pyunicore.client.Client: 39 | return pyunicore.client.Client(credential, site_url=api_url) 40 | 41 | 42 | def _authentication_failed(client: pyunicore.client.Client) -> bool: 43 | return False if client.properties["client"]["xlogin"] else True 44 | -------------------------------------------------------------------------------- /pyunicore/helpers/jobs/__init__.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.jobs.data import Credentials 2 | from pyunicore.helpers.jobs.data import Export 3 | from pyunicore.helpers.jobs.data import Import 4 | from pyunicore.helpers.jobs.description import Description 5 | from pyunicore.helpers.jobs.resources import Resources 6 | -------------------------------------------------------------------------------- /pyunicore/helpers/jobs/data.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import Optional 4 | 5 | from pyunicore.helpers import _api_object 6 | 7 | 8 | @dataclasses.dataclass 9 | class Credentials(_api_object.ApiRequestObject): 10 | """Credentials for an external service a file might be imported from. 11 | 12 | Args: 13 | username (str): User name. 14 | password (str): Password. 15 | 16 | """ 17 | 18 | username: str 19 | password: str 20 | 21 | def _to_dict(self) -> Dict[str, str]: 22 | return { 23 | "Username": self.username, 24 | "Password": self.password, 25 | } 26 | 27 | 28 | @dataclasses.dataclass 29 | class Import(_api_object.ApiRequestObject): 30 | """An import.""" 31 | 32 | from_: str 33 | to: str 34 | fail_on_error: bool = True 35 | data: Optional[str] = None 36 | credentials: Optional[Credentials] = None 37 | 38 | def _to_dict(self) -> Dict: 39 | return { 40 | "From": self.from_, 41 | "To": self.to, 42 | "FailOnError": self.fail_on_error, 43 | "Data": self.data, 44 | "Credentials": self.credentials, 45 | } 46 | 47 | 48 | @dataclasses.dataclass 49 | class Export(_api_object.ApiRequestObject): 50 | """An export.""" 51 | 52 | from_: str 53 | to: str 54 | 55 | def _to_dict(self) -> Dict[str, str]: 56 | return { 57 | "From": self.from_, 58 | "To": self.to, 59 | } 60 | -------------------------------------------------------------------------------- /pyunicore/helpers/jobs/description.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import List 4 | from typing import Optional 5 | 6 | from pyunicore.helpers import _api_object 7 | from pyunicore.helpers.jobs import data 8 | from pyunicore.helpers.jobs import resources as _resources 9 | 10 | 11 | @dataclasses.dataclass 12 | class Description(_api_object.ApiRequestObject): 13 | """UNICORE's job description for submitting jobs. 14 | 15 | Args: 16 | executable (str, optional) : Command line. 17 | project (str, optional): Accounting project. 18 | resources (Resources, optional): The job's resource requests. 19 | application_name (str, optional): Application name. 20 | application_version (str, optional): Application version. 21 | arguments (list[str], optional): Command line arguments. 22 | environment (dict[str, str], optinal): Environment values. 23 | parameters (str, optional): Application parameters. 24 | stdout (str, default="stdout"): Filename for the standard output. 25 | stderr (str, default="stderr"): Filename for the standard error. 26 | stdin (str, optional): Filename for the standard input. 27 | ignore_non_zero_exit_code (bool, default=False): Don't fail the job if 28 | app exits with non-zero exit code. 29 | user_precommand (str, optional): Pre-processing. 30 | run_user_precommand_on_login_node (bool, default=True): Pre-processing 31 | is done on login node. 32 | user_precommand_ignore_non_zero_exit_code (bool, default=False): Don't 33 | fail job if pre-command fails. 34 | user_postommandFalse: Post-processing. 35 | run_user_postcommand_on_login_node (bool, default=True): Post-processing 36 | is done on login node. 37 | user_postcommand_ignore_non_zero_exit_code (bool, default=False): Don't 38 | fail job if post-command fails. 39 | imports (list[Import], optional): Stage-in / data import. 40 | exports (list[Export], optional): Stage-out / data export. 41 | have_client_stage_in (bool, default=False): Tell the server that the 42 | client does / does not want to send any additional files. 43 | job_type (str, default="batch): 'batch', 'on_login_node', 'raw', 'allocate' 44 | Whether to run the job via the batch system ('batch', default) or 45 | on a login node ('interactive'), or as a batch job but with a 46 | user-specified file containing the batch system directives ('raw'). 47 | The 'allocate' job type will only create an allocation, 48 | without running anything. 49 | login_node (str, optional): For jobs of the 'on_login_node' type, select 50 | a login node (by name, as configured server side. 51 | Wildcards '*' and '?' can be used). 52 | bss_file (str, optional): For 'raw' jobs, specify the relative or 53 | absolute file name of a file containing batch system directives. 54 | UNICORE will append the user executable. 55 | tags (list[str], optional): Job tags. 56 | notification (str, optional): URL to send job status change 57 | notifications to. Will be sent via HTTP POST. 58 | user_email (str, optional): User email to send notifications to 59 | Only works if the batch system supports it. 60 | name (str, optional): Job name. 61 | """ 62 | 63 | executable: Optional[str] = None 64 | project: Optional[str] = None 65 | resources: Optional[_resources.Resources] = dataclasses.field( 66 | default_factory=_resources.Resources 67 | ) 68 | application_name: Optional[str] = None 69 | application_version: Optional[str] = None 70 | arguments: Optional[List[str]] = None 71 | environment: Optional[Dict[str, str]] = None 72 | parameters: Optional[Dict[str, str]] = None 73 | stdout: Optional[str] = "stdout" 74 | stderr: Optional[str] = "stderr" 75 | stdin: Optional[str] = None 76 | ignore_non_zero_exit_code: Optional[bool] = False 77 | user_precommand: Optional[str] = None 78 | run_user_precommand_on_login_node: Optional[bool] = True 79 | user_precommand_ignore_non_zero_exitcode: Optional[bool] = False 80 | user_postcommand: Optional[str] = None 81 | run_user_postcommand_on_login_node: Optional[bool] = True 82 | user_postcommand_ignore_non_zero_exit_code: Optional[bool] = False 83 | imports: Optional[List[data.Import]] = None 84 | exports: Optional[List[data.Export]] = None 85 | have_client_stage_in: Optional[bool] = False 86 | job_type: Optional[str] = "normal" 87 | login_node: Optional[str] = None 88 | bss_file: Optional[str] = None 89 | tags: Optional[List[str]] = None 90 | notification: Optional[str] = None 91 | user_email: Optional[str] = None 92 | name: Optional[str] = None 93 | 94 | def __post_init__(self): 95 | """Set `have_client_stage_in=True` if any files have to be imported.""" 96 | if self.imports: 97 | self.have_client_stage_in = True 98 | 99 | if self.job_type == "raw" and self.bss_file is None: 100 | raise ValueError("If job type is 'raw', BSS file has to be specified") 101 | 102 | def _to_dict(self) -> Dict: 103 | return { 104 | "ApplicationName": self.application_name, 105 | "ApplicationVersion": self.application_version, 106 | "Executable": self.executable, 107 | "Arguments": self.arguments, 108 | "Environment": self.environment, 109 | "Parameters": self.parameters, 110 | "Stdout": self.stdout, 111 | "Stderr": self.stderr, 112 | "Stdin": self.stdin, 113 | "IgnoreNonZeroExitCode": self.ignore_non_zero_exit_code, 114 | "User precommand": self.user_precommand, 115 | "RunUserPrecommandOnLoginNode": (self.run_user_precommand_on_login_node), 116 | "UserPrecommandIgnoreNonZeroExitcode": (self.user_precommand_ignore_non_zero_exitcode), 117 | "User postcommand": self.user_postcommand, 118 | "RunUserPostcommandOnLoginNode": (self.run_user_postcommand_on_login_node), 119 | "UserPostcommandIgnoreNonZeroExitcode": ( 120 | self.user_postcommand_ignore_non_zero_exit_code 121 | ), 122 | "Project": self.project, 123 | "Resources": self.resources, 124 | "Imports": self.imports, 125 | "Exports": self.exports, 126 | "haveClientStageIn": self.have_client_stage_in, 127 | "Job type": self.job_type, 128 | "Login node": self.login_node, 129 | "BSS file": self.bss_file, 130 | "Tags": self.tags, 131 | "Notification": self.notification, 132 | "User email": self.user_email, 133 | "Name": self.name, 134 | } 135 | -------------------------------------------------------------------------------- /pyunicore/helpers/jobs/resources.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import Optional 4 | 5 | from pyunicore.helpers import _api_object 6 | 7 | 8 | @dataclasses.dataclass 9 | class Resources(_api_object.ApiRequestObject): 10 | """Resources to request on the remote system. 11 | 12 | Args: 13 | runtime (str, optional): Job runtime (wall time). 14 | In seconds by default, append "min", "h", or "d" for other units. 15 | queue (str, optional): Batch system queue/partition to use. 16 | nodes (int, optional): Number of nodes. 17 | cpus (int, optional): Total number of CPUs. 18 | cpus_per_node (int, optional): Number of CPUs per node. 19 | memory (str, optional): Memory per node. 20 | reservation (str, optional): Batch system reservation ID. 21 | node_constraints (str, optional): Batch system node constraints. 22 | qos (str, optional): Batch system QoS. 23 | 24 | """ 25 | 26 | runtime: Optional[str] = None 27 | queue: Optional[str] = None 28 | nodes: Optional[int] = None 29 | cpus: Optional[int] = None 30 | cpus_per_node: Optional[int] = None 31 | memory: Optional[str] = None 32 | reservation: Optional[str] = None 33 | node_constraints: Optional[str] = None 34 | qos: Optional[str] = None 35 | 36 | def _to_dict(self) -> Dict: 37 | return { 38 | "Runtime": self.runtime, 39 | "Queue": self.queue, 40 | "Nodes": self.nodes, 41 | "CPUs": self.cpus, 42 | "CPUsPerNode": self.cpus_per_node, 43 | "Memory": self.memory, 44 | "Reservation": self.reservation, 45 | "NodeConstraints": self.node_constraints, 46 | "QoS": self.qos, 47 | } 48 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/__init__.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows import activities 2 | from pyunicore.helpers.workflows.description import Description 3 | from pyunicore.helpers.workflows.transition import Transition 4 | from pyunicore.helpers.workflows.variable import Variable 5 | from pyunicore.helpers.workflows.variable import VariableType 6 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/__init__.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows.activities import loops 2 | from pyunicore.helpers.workflows.activities.activity import Branch 3 | from pyunicore.helpers.workflows.activities.activity import Hold 4 | from pyunicore.helpers.workflows.activities.activity import Merge 5 | from pyunicore.helpers.workflows.activities.activity import Split 6 | from pyunicore.helpers.workflows.activities.activity import Start 7 | from pyunicore.helpers.workflows.activities.activity import Synchronize 8 | from pyunicore.helpers.workflows.activities.job import jobs 9 | from pyunicore.helpers.workflows.activities.modify_variable import ModifyVariable 10 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/activity.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import dataclasses 3 | from typing import Dict 4 | 5 | from pyunicore.helpers import _api_object 6 | 7 | 8 | @dataclasses.dataclass 9 | class Activity(_api_object.ApiRequestObject): 10 | """UNICORE's activity description for submitting workflows. 11 | 12 | Args: 13 | id (str): ID of the activity. 14 | Must be unique within the workflow. 15 | 16 | """ 17 | 18 | id: str 19 | 20 | @property 21 | def type(self) -> str: 22 | """Return the UNICORE type of the activity.""" 23 | return self._type() 24 | 25 | @abc.abstractmethod 26 | def _type(self) -> str: 27 | """Return the UNICORE type of the activity.""" 28 | 29 | def _to_dict(self) -> Dict: 30 | return { 31 | "id": self.id, 32 | "type": self.type, 33 | **self._activity_to_dict(), 34 | } 35 | 36 | @abc.abstractmethod 37 | def _activity_to_dict(self) -> Dict: 38 | """Return dict for the respective activity type.""" 39 | 40 | 41 | @dataclasses.dataclass 42 | class Start(Activity): 43 | """A start activity within a workflow. 44 | 45 | Denotes an explicit start activity. If no such activity is present, the 46 | processing engine will try to detect the proper starting activities. 47 | 48 | """ 49 | 50 | def _type(self) -> str: 51 | return "START" 52 | 53 | def _activity_to_dict(self) -> Dict: 54 | return {} 55 | 56 | 57 | @dataclasses.dataclass 58 | class Split(Activity): 59 | """A split activity within a workflow. 60 | 61 | This activity can have multiple outgoing transitions. All transitions with 62 | matching conditions will be followed. This is comparable to an 63 | "if() … if() … if()" construct in a programming language. 64 | 65 | """ 66 | 67 | def _type(self) -> str: 68 | return "Split" 69 | 70 | def _activity_to_dict(self) -> Dict: 71 | return {} 72 | 73 | 74 | @dataclasses.dataclass 75 | class Branch(Activity): 76 | """A start activity within a workflow. 77 | 78 | This activity can have multiple outgoing transitions. The transition with 79 | the first matching condition will be followed. This is comparable to an 80 | "if() … elseif() … else()" construct in a programming language. 81 | 82 | """ 83 | 84 | def _type(self) -> str: 85 | return "BRANCH" 86 | 87 | def _activity_to_dict(self) -> Dict: 88 | return {} 89 | 90 | 91 | @dataclasses.dataclass 92 | class Merge(Activity): 93 | """A start activity within a workflow. 94 | 95 | Denotes an explicit start activity. If no such activity is present, the 96 | processing engine will try to detect the proper starting activities. 97 | 98 | """ 99 | 100 | def _type(self) -> str: 101 | return "Merge" 102 | 103 | def _activity_to_dict(self) -> Dict: 104 | return {} 105 | 106 | 107 | @dataclasses.dataclass 108 | class Synchronize(Activity): 109 | """A start activity within a workflow. 110 | 111 | Merges multiple flows and synchronises them. 112 | 113 | """ 114 | 115 | def _type(self) -> str: 116 | return "Synchronize" 117 | 118 | def _activity_to_dict(self) -> Dict: 119 | return {} 120 | 121 | 122 | @dataclasses.dataclass 123 | class Hold(Activity): 124 | """A hold activity within a workflow. 125 | 126 | Stops further processing of the current flow until the client explicitely 127 | sends continue message. 128 | 129 | """ 130 | 131 | def _type(self) -> str: 132 | return "HOLD" 133 | 134 | def _activity_to_dict(self) -> Dict: 135 | return {} 136 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/job.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import dataclasses 3 | from typing import Dict 4 | from typing import List 5 | from typing import Optional 6 | from typing import Type 7 | 8 | from pyunicore.helpers import _api_object 9 | from pyunicore.helpers import jobs 10 | from pyunicore.helpers.workflows.activities import activity 11 | 12 | 13 | class Option: 14 | """An activity option for jobs in a workflow.""" 15 | 16 | class _Option(_api_object.ApiRequestObject): 17 | value: Type 18 | 19 | def _to_dict(self) -> Dict: 20 | return {self._name: self.value} 21 | 22 | @property 23 | @abc.abstractmethod 24 | def _name(self) -> str: 25 | """Return the UNICORE name of the option.""" 26 | 27 | @dataclasses.dataclass 28 | class IgnoreFailure(_Option): 29 | """Whether to ignore any failure of the task. 30 | 31 | Args: 32 | value (bool): Whether to ignore the failure. 33 | The workflow engine continues processing as if the activity had 34 | been completed successfully. This has nothing to do with the 35 | exit code of the actual UNICORE job! Failure means for example 36 | data staging failed, or no matching target system for the job 37 | could be found. 38 | 39 | """ 40 | 41 | value: bool 42 | 43 | @property 44 | def _name(self) -> str: 45 | return "IGNORE_FAILURE" 46 | 47 | @dataclasses.dataclass 48 | class MaxResubmits(_Option): 49 | """Number of times the activity will be retried. 50 | 51 | By default, the workflow engine will re-try three times. 52 | 53 | """ 54 | 55 | value: int 56 | 57 | @property 58 | def _name(self) -> str: 59 | return "MAX_RESUBMITS" 60 | 61 | 62 | @dataclasses.dataclass 63 | class UserPreferences(_api_object.ApiRequestObject): 64 | """User preferences.""" 65 | 66 | role: str 67 | uid: str 68 | group: str 69 | supplementary_groups: str 70 | 71 | def _to_dict(self) -> dict: 72 | return { 73 | "role": self.role, 74 | "uid": self.uid, 75 | "group": self.group, 76 | "supplementaryGroups": self.supplementary_groups, 77 | } 78 | 79 | 80 | @dataclasses.dataclass 81 | class Job(activity.Activity): 82 | """A job activity within a workflow. 83 | 84 | Denotes a executable (job) activity. In this case, the job sub element 85 | holds the JSON job definition. (If a "job" element is present, you may 86 | leave out the "type".) 87 | 88 | Args: 89 | job (JobDescription): Description of the job. 90 | site_name (str): Name of the site to execute the job on. 91 | user_preferences (UserPreferences, optional): User preferences to pass. 92 | options (list[JobOption], optional): Options to pass. 93 | 94 | """ 95 | 96 | description: jobs.Description 97 | site_name: str 98 | user_preferences: Optional[UserPreferences] = None 99 | options: Optional[List[Option]] = None 100 | 101 | def _type(self) -> str: 102 | return "JOB" 103 | 104 | def _activity_to_dict(self) -> Dict: 105 | if self.options is not None: 106 | options = {k: v for o in self.options for k, v in o.to_dict().items()} 107 | else: 108 | options = None 109 | 110 | return { 111 | "job": { 112 | **self.description.to_dict(), 113 | "Site name": self.site_name, 114 | "User preferences": self.user_preferences, 115 | }, 116 | "options": options, 117 | } 118 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/loops/__init__.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows.activities.loops.body import Body 2 | from pyunicore.helpers.workflows.activities.loops.for_loop import Chunking 3 | from pyunicore.helpers.workflows.activities.loops.for_loop import ChunkingType 4 | from pyunicore.helpers.workflows.activities.loops.for_loop import File 5 | from pyunicore.helpers.workflows.activities.loops.for_loop import Files 6 | from pyunicore.helpers.workflows.activities.loops.for_loop import ForEach 7 | from pyunicore.helpers.workflows.activities.loops.for_loop import Values 8 | from pyunicore.helpers.workflows.activities.loops.for_loop import Variable 9 | from pyunicore.helpers.workflows.activities.loops.for_loop import Variables 10 | from pyunicore.helpers.workflows.activities.loops.repeat_until_loop import RepeatUntil 11 | from pyunicore.helpers.workflows.activities.loops.while_loop import While 12 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/loops/_loop.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | from pyunicore.helpers.workflows.activities import activity 4 | from pyunicore.helpers.workflows.activities.loops import body 5 | 6 | 7 | @dataclasses.dataclass 8 | class Loop(activity.Activity): 9 | """A loop-like activity within a workflow. 10 | 11 | Args: 12 | body (Body): Loop body. 13 | 14 | """ 15 | 16 | body: body.Body 17 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/loops/body.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import List 4 | 5 | from pyunicore.helpers import _api_object 6 | from pyunicore.helpers.workflows import transition 7 | from pyunicore.helpers.workflows.activities import activity 8 | 9 | 10 | @dataclasses.dataclass 11 | class Body(_api_object.ApiRequestObject): 12 | """Body of a loop.""" 13 | 14 | activities: List[activity.Activity] 15 | transitions: List[transition.Transition] 16 | condition: str 17 | 18 | def _to_dict(self) -> Dict: 19 | return { 20 | "activities": self.activities, 21 | "transitions": self.transitions, 22 | "condition": self.condition, 23 | } 24 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/loops/for_loop.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import List 4 | from typing import Optional 5 | from typing import Union 6 | 7 | from pyunicore.helpers import _api_object 8 | from pyunicore.helpers.workflows import variable 9 | from pyunicore.helpers.workflows.activities.loops import _loop 10 | 11 | 12 | @dataclasses.dataclass 13 | class Values(_api_object.ApiRequestObject): 14 | """A range of values to iterate over.""" 15 | 16 | values: List 17 | 18 | def _to_dict(self) -> Dict: 19 | return {"values": self.values} 20 | 21 | 22 | @dataclasses.dataclass 23 | class Variable(variable.Variable): 24 | """A variable to use in an iteration. 25 | 26 | Args: 27 | expression (str): Expression to evaluate at each iteration. 28 | end_condition (str): Condition when to end the iteration. 29 | 30 | """ 31 | 32 | expression: str 33 | end_condition: str 34 | 35 | def _to_dict(self) -> Dict: 36 | return { 37 | "variable_name": self.name, 38 | "type": variable.VariableType.get_type_name(self.type), 39 | "start_value": self.initial_value, 40 | "expression": self.expression, 41 | "end_condition": self.end_condition, 42 | } 43 | 44 | 45 | @dataclasses.dataclass 46 | class Variables(_api_object.ApiRequestObject): 47 | """A set of variables to iterate over.""" 48 | 49 | variables: List[Variable] 50 | 51 | def _to_dict(self) -> Dict: 52 | return {"variables": self.variables} 53 | 54 | 55 | @dataclasses.dataclass 56 | class File(_api_object.ApiRequestObject): 57 | """A file configuration to include in an iteration. 58 | 59 | Args: 60 | base (str): Base of the filenames, which will be resolved at runtime. 61 | include (list[str]): List if file names or regular expressions. 62 | exclude (list[str]): List if file names or regular expressions. 63 | recurse (bool, default=False): Whether the resolution should be done 64 | recursively into any subdirectories. 65 | indirection (bool, default=False): Whether to load the given file(s) 66 | at runtime. 67 | 68 | 69 | """ 70 | 71 | base: str 72 | include: List[str] 73 | exclude: List[str] 74 | recurse: bool = False 75 | indirection: bool = False 76 | 77 | def _to_dict(self) -> Dict: 78 | return { 79 | "base": self.base, 80 | "include": self.include, 81 | "exclude": self.exclude, 82 | "recurse": self.recurse, 83 | "indirection": self.indirection, 84 | } 85 | 86 | 87 | @dataclasses.dataclass 88 | class Files(_api_object.ApiRequestObject): 89 | """A set of files to iterator over.""" 90 | 91 | files: List[File] 92 | 93 | def _to_dict(self) -> Dict: 94 | return {"file_sets": self.files} 95 | 96 | 97 | class ChunkingType: 98 | """The type of the chunks. 99 | 100 | Attrs: 101 | Normal (str): Number of files to use as chunks. 102 | Size (str): Size in kbytes to process per chunk. 103 | 104 | """ 105 | 106 | Normal = "NORMAL" 107 | Size = "SIZE" 108 | 109 | 110 | @dataclasses.dataclass 111 | class Chunking(_api_object.ApiRequestObject): 112 | """A chunking configuration to use in an iteration. 113 | 114 | Args: 115 | chunksize(int): Size of the chunks. 116 | chunksize_formula (str, optional): Expression to use to calculate the 117 | chunksize at runtime. 118 | type (ChunkingType, default=Normal): Type of the `chunksize`. 119 | - `ChunkingType.Normal`: Number of files in a chunk. 120 | - `ChunkingType.Size`: Total size of a chunk in kbytes. 121 | filename_format (str): Allows to control how the individual files 122 | should be named. 123 | 124 | Notes: 125 | Either `chunksize` or `chunksize_formula` must be given. 126 | 127 | """ 128 | 129 | chunksize: Optional[int] = None 130 | chunksize_formula: Optional[str] = None 131 | type: ChunkingType = ChunkingType.Normal 132 | filename_format: Optional[str] = None 133 | 134 | def __post_init__(self): 135 | """Check that either chunksize or formula is given.""" 136 | if (self.chunksize is None and self.chunksize_formula is None) or ( 137 | self.chunksize is not None and self.chunksize_formula is not None 138 | ): 139 | raise ValueError("Either `chunksize` or `chunksize_formula` must be given") 140 | 141 | def _to_dict(self) -> Dict: 142 | return { 143 | "chunksize": self.chunksize, 144 | "type": self.type, 145 | "filename_format": self.filename_format, 146 | "chunksize_formula": self.chunksize_formula, 147 | } 148 | 149 | 150 | @dataclasses.dataclass 151 | class ForEach(_loop.Loop): 152 | """A for-each-loop-like activity within a workflow. 153 | 154 | Args: 155 | range (Values, Variables or Files): Range to iterator over. 156 | iterator_name (str, default="IT"): Name of the iterator. 157 | chunking (Chunking): Chunking to use for the range. 158 | 159 | """ 160 | 161 | range: Union[Values, Variables, Files] 162 | iterator_name: str = "IT" 163 | chunking: Optional[Chunking] = None 164 | 165 | def _type(self) -> str: 166 | return "FOR_EACH" 167 | 168 | def _activity_to_dict(self) -> Dict: 169 | return { 170 | "iterator_name": self.iterator_name, 171 | "body": self.body, 172 | "chunking": self.chunking, 173 | **self.range.to_dict(), 174 | } 175 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/loops/repeat_until_loop.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | from pyunicore.helpers.workflows.activities.loops import while_loop 4 | 5 | 6 | @dataclasses.dataclass 7 | class RepeatUntil(while_loop.While): 8 | """A repeat-until-loop-like activity within a workflow.""" 9 | 10 | def _type(self) -> str: 11 | return "REPEAT_UNTIL" 12 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/loops/while_loop.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import List 4 | 5 | from pyunicore.helpers.workflows import variable 6 | from pyunicore.helpers.workflows.activities.loops import _loop 7 | 8 | 9 | @dataclasses.dataclass 10 | class While(_loop.Loop): 11 | """A while-loop-like activity within a workflow. 12 | 13 | Args: 14 | variables (list[Variable]): Variables to use in the loop. 15 | 16 | """ 17 | 18 | variables: List[variable.Variable] 19 | 20 | def _type(self) -> str: 21 | return "WHILE" 22 | 23 | def _activity_to_dict(self) -> Dict: 24 | return { 25 | "variables": self.variables, 26 | "body": self.body, 27 | } 28 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/activities/modify_variable.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | 4 | from pyunicore.helpers.workflows.activities import activity 5 | 6 | 7 | @dataclasses.dataclass 8 | class ModifyVariable(activity.Activity): 9 | """Modifies a variable within the workflow. 10 | 11 | Allows to modify a workflow variable. An option named "variableName" 12 | identifies the variable to be modified, and an option "expression" holds 13 | the modification expression in the Groovy programming language syntax. 14 | 15 | Args: 16 | variable_name (str): name of the variable to modify. 17 | expression (str): Groovy-syntax expression to modify the variable. 18 | 19 | """ 20 | 21 | variable_name: str 22 | expression: str 23 | 24 | def _type(self) -> str: 25 | return "ModifyVariable" 26 | 27 | def _activity_to_dict(self) -> Dict: 28 | return { 29 | "variableName": self.variable_name, 30 | "expression": self.expression, 31 | } 32 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/description.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import List 4 | from typing import Optional 5 | 6 | from pyunicore.helpers import _api_object 7 | from pyunicore.helpers.workflows import transition 8 | from pyunicore.helpers.workflows import variable 9 | from pyunicore.helpers.workflows.activities import activity 10 | 11 | 12 | @dataclasses.dataclass 13 | class Description(_api_object.ApiRequestObject): 14 | """UNICORE's workflow description for submitting workflows. 15 | 16 | Args: 17 | activities (list): 18 | subworkflows (list(Description), optional): 19 | transitions (list): 20 | variables (list): 21 | notification (str, optional): URL to send notifications to. 22 | The UNICORE Workflow server will send a POST notification when the 23 | workflow has finished processing. Notifcation messages have the 24 | following content: 25 | ```JSON 26 | { 27 | "href" : "workflow_url", 28 | "group_id": "id of the workflow or sub-workflow", 29 | "status": "...", 30 | "statusMessage": "..." 31 | } 32 | ``` 33 | tags (list, optional): tags for filtering the list of workflows. 34 | 35 | """ 36 | 37 | activities: List[activity.Activity] 38 | transitions: List[transition.Transition] 39 | variables: List[variable.Variable] 40 | subworkflows: Optional[List["Description"]] = None 41 | notification: Optional[str] = None 42 | tags: Optional[List[str]] = None 43 | 44 | def _to_dict(self) -> Dict: 45 | return { 46 | "activities": self.activities, 47 | "subworkflows": self.subworkflows, 48 | "transitions": self.transitions, 49 | "variables": self.variables, 50 | "notification": self.notification, 51 | "tags": self.tags, 52 | } 53 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/transition.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | from typing import Optional 4 | 5 | from pyunicore.helpers import _api_object 6 | 7 | 8 | @dataclasses.dataclass 9 | class Transition(_api_object.ApiRequestObject): 10 | """UNICORE's transition description for submitting workflows. 11 | 12 | Args: 13 | from_ (str): ID of the activity or subworkflow. 14 | to (str): ID of the activity or subworkflow. 15 | condition (str, optional): Transition is only followed if this 16 | evaluates to true. 17 | 18 | """ 19 | 20 | from_: str 21 | to: str 22 | condition: Optional[str] = None 23 | 24 | def _to_dict(self) -> Dict: 25 | return { 26 | "from": self.from_, 27 | "to": self.to, 28 | "condition": self.condition, 29 | } 30 | -------------------------------------------------------------------------------- /pyunicore/helpers/workflows/variable.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Any 3 | from typing import Dict 4 | from typing import Tuple 5 | from typing import Type 6 | from typing import Union 7 | 8 | from pyunicore.helpers import _api_object 9 | 10 | 11 | class VariableType: 12 | """Accepted variable types for workflow variables.""" 13 | 14 | String = str 15 | Integer = int 16 | Float = float 17 | Boolean = bool 18 | 19 | @classmethod 20 | def get_types(cls) -> Tuple[Type]: 21 | """Return all available types.""" 22 | return tuple(cls._types().keys()) 23 | 24 | @classmethod 25 | def get_type_name(cls, type: Type) -> str: 26 | """Get the UNICORE name for the type.""" 27 | return cls._types()[type] 28 | 29 | @classmethod 30 | def _types(cls) -> Dict[Type, str]: 31 | return { 32 | cls.String: "STRING", 33 | cls.Integer: "INTEGER", 34 | cls.Float: "FLOAT", 35 | cls.Boolean: "BOOLEAN", 36 | } 37 | 38 | 39 | @dataclasses.dataclass 40 | class Variable(_api_object.ApiRequestObject): 41 | """UNICORE's variable description for submitting workflows. 42 | 43 | Args: 44 | name (str): Name of the variable. 45 | type (VariableType): Type of the variable. 46 | initial_value: Initial value of the variable. 47 | 48 | """ 49 | 50 | name: str 51 | type: Union[VariableType, Any] 52 | initial_value: Any 53 | 54 | def __post_init__(self) -> None: 55 | self._check_for_correct_type() 56 | self._check_initial_value_for_correct_type() 57 | 58 | def _check_for_correct_type(self): 59 | allowed_types = VariableType.get_types() 60 | if self.type not in allowed_types: 61 | raise ValueError( 62 | f"{self.type} is not a valid variable type. " 63 | f"Allowed variable types: {allowed_types}" 64 | ) 65 | 66 | def _check_initial_value_for_correct_type(self) -> None: 67 | if not isinstance(self.initial_value, self.type): 68 | actual_type = type(self.initial_value) 69 | raise ValueError( 70 | f"Initial value of {self} has incorrect type " 71 | f"{actual_type}, expected {self.type}" 72 | ) 73 | 74 | def _to_dict(self) -> Dict: 75 | return { 76 | "name": self.name, 77 | "type": VariableType.get_type_name(self.type), 78 | "initial_value": self._convert_value(), 79 | } 80 | 81 | def _convert_value(self) -> Any: 82 | if self.type == VariableType.Boolean: 83 | return str(self.initial_value).lower() 84 | return self.initial_value 85 | -------------------------------------------------------------------------------- /pyunicore/uftp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HumanBrainProject/pyunicore/bbde277261c3a25757df0942a5aa41bf3e49d933/pyunicore/uftp/__init__.py -------------------------------------------------------------------------------- /pyunicore/uftp/uftp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import stat 3 | from ftplib import FTP 4 | from sys import maxsize 5 | from time import localtime 6 | from time import mktime 7 | from time import strftime 8 | from time import strptime 9 | 10 | from pyunicore.client import Transport 11 | from pyunicore.credentials import Credential 12 | 13 | 14 | class UFTP: 15 | """ 16 | Authenticate UFTP sessions via Authserver 17 | Uses ftplib to open the session and interact with the UFTPD server 18 | """ 19 | 20 | uftp_session_tag = "___UFTP___MULTI___FILE___SESSION___MODE___" 21 | 22 | def __init__(self): 23 | self.ftp = None 24 | self.uid = os.getuid() 25 | self.gid = os.getgid() 26 | 27 | def connect(self, security, base_url, base_dir=""): 28 | """authenticate and open a UFTP session""" 29 | host, port, password = self.authenticate(security, base_url, base_dir) 30 | self.open_uftp_session(host, port, password) 31 | 32 | def open_uftp_session(self, host, port, password): 33 | """open an FTP session at the given UFTP server""" 34 | self.ftp = FTP() 35 | self.ftp.connect(host, port) 36 | self.ftp.login("anonymous", password) 37 | 38 | def authenticate(self, security, base_url, base_dir="", preferences=None, persistent=True): 39 | """authenticate to the auth server and return a tuple (host, port, one-time-password)""" 40 | if isinstance(security, Credential): 41 | transport = Transport(security) 42 | elif isinstance(security, Transport): 43 | transport = security._clone() 44 | else: 45 | raise TypeError("Need Credential or Transport object") 46 | if base_dir != "" and not base_dir.endswith("/"): 47 | base_dir += "/" 48 | if preferences is not None: 49 | transport.preferences = preferences 50 | req = { 51 | "serverPath": base_dir + self.uftp_session_tag, 52 | } 53 | if persistent: 54 | req["persistent"] = "true" 55 | params = transport.post(url=base_url, json=req).json() 56 | return params["serverHost"], params["serverPort"], params["secret"] 57 | 58 | __perms = {"r": stat.S_IRUSR, "w": stat.S_IWUSR, "x": stat.S_IXUSR} 59 | __type = {"file": stat.S_IFREG, "dir": stat.S_IFDIR} 60 | 61 | def normalize(self, path): 62 | if path is not None: 63 | if path.startswith("/"): 64 | path = path[1:] 65 | return path 66 | 67 | def stat(self, path): 68 | """get os.stat() style info about a remote file/directory""" 69 | path = self.normalize(path) 70 | self.ftp.putline("MLST %s" % path) 71 | lines = self.ftp.getmultiline().split("\n") 72 | if len(lines) != 3 or not lines[0].startswith("250"): 73 | raise OSError("File not found. Server reply: %s " % str(lines[0])) 74 | infos = lines[1].strip().split(" ")[0].split(";") 75 | raw_info = {} 76 | for x in infos: 77 | tok = x.split("=") 78 | if len(tok) != 2: 79 | continue 80 | raw_info[tok[0]] = tok[1] 81 | st = {} 82 | st["st_size"] = int(raw_info["size"]) 83 | st["st_uid"] = self.uid 84 | st["st_gid"] = self.gid 85 | mode = UFTP.__type[raw_info.get("type", stat.S_IFREG)] 86 | for x in raw_info["perm"]: 87 | mode = mode | UFTP.__perms.get(x, stat.S_IRUSR) 88 | st["st_mode"] = mode 89 | if raw_info.get("UNIX.mode", None) is not None: 90 | st["st_mode"] = int(raw_info["UNIX.mode"], 8) 91 | ttime = int(mktime(strptime(raw_info["modify"], "%Y%m%d%H%M%S"))) 92 | st["st_mtime"] = ttime 93 | st["st_atime"] = ttime 94 | return st 95 | 96 | def listdir(self, directory): 97 | """return a list of files in the given directory""" 98 | listing = [] 99 | directory = self.normalize(directory) 100 | self.ftp.retrlines(cmd="LIST %s" % directory, callback=listing.append) 101 | return [x.split(" ")[-1] for x in listing] 102 | 103 | def mkdir(self, directory, mode): 104 | directory = self.normalize(directory) 105 | self.ftp.voidcmd("MKD %s" % directory) 106 | 107 | def rmdir(self, directory): 108 | directory = self.normalize(directory) 109 | self.ftp.voidcmd("RMD %s" % directory) 110 | 111 | def rm(self, path): 112 | path = self.normalize(path) 113 | self.ftp.voidcmd("DELE %s" % path) 114 | 115 | def rename(self, source, target): 116 | source = self.normalize(source) 117 | target = self.normalize(target) 118 | reply = self.ftp.sendcmd("RNFR %s" % source) 119 | if not reply.startswith("350"): 120 | raise OSError("Could not rename: " % reply) 121 | self.ftp.voidcmd("RNTO %s" % target) 122 | 123 | def set_time(self, path, mtime): 124 | path = self.normalize(path) 125 | stime = strftime("%Y%m%d%H%M%S", localtime(mtime)) 126 | reply = self.ftp.sendcmd(f"MFMT {stime} {path}") 127 | if not reply.startswith("213"): 128 | raise OSError("Could not set time: " % reply) 129 | 130 | def chmod(self, path, mode): 131 | path = self.normalize(path) 132 | reply = self.ftp.sendcmd(f"MFF UNIX.mode={oct(mode)[2:]}; {path}") 133 | if not reply.startswith("213"): 134 | raise OSError("Could not chmod: " % reply) 135 | 136 | def close(self): 137 | if self.ftp is not None: 138 | self.ftp.close() 139 | 140 | def _send_range(self, offset, length, rfc=False): 141 | end_byte = offset + length - 1 if rfc else offset + length 142 | self.ftp.sendcmd(f"RANG {offset} {end_byte}") 143 | 144 | def get_write_socket(self, path, offset): 145 | path = self.normalize(path) 146 | if offset > 0: 147 | self._send_range(offset, maxsize) 148 | else: 149 | self.ftp.sendcmd(f"ALLO {maxsize}") 150 | return self.ftp.transfercmd("STOR %s" % path) 151 | 152 | def get_read_socket(self, path, offset): 153 | path = self.normalize(path) 154 | return self.ftp.transfercmd("RETR %s" % path, rest=offset) 155 | -------------------------------------------------------------------------------- /pyunicore/uftp/uftpfs.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from os import getenv 3 | 4 | from fs.ftpfs import FTPFS 5 | from fs.opener import Opener 6 | 7 | import pyunicore.credentials as uc_credentials 8 | from pyunicore.uftp.uftp import UFTP 9 | 10 | 11 | class UFTPFS(FTPFS): 12 | """A UFTP (UNICORE FTP) Filesystem. 13 | 14 | This extends the fs.ftpfs.FTPFS filesystem 15 | with UFTP authentication. 16 | 17 | Example: create with auth URL and username/password credentials 18 | 19 | from pyunicore.credentials import UsernamePassword 20 | from pyunicore.uftp.uftpfs import UFTPFS 21 | 22 | auth = "https://localhost:9000/rest/auth/TEST" 23 | creds = UsernamePassword("demouser", "test123") 24 | base_path = "/" 25 | 26 | uftp_fs = UFTPFS(auth, creds, base_path) 27 | uftp_fs.tree() 28 | 29 | Example: create with URL 30 | 31 | from fs import open_fs 32 | fs_url = "uftp://demouser:test123@localhost:9000/rest/auth/TEST:/opt/shared-data" 33 | uftp_fs = open_fs(fs_url) 34 | uftp_fs.tree() 35 | 36 | 37 | """ 38 | 39 | def __init__(self, auth_url, creds, base_path="/"): 40 | """Creates a new UFTP FS instance authenticating using the given URL and credentials""" 41 | self.uftp_host, self.uftp_port, self.uftp_password = UFTP().authenticate( 42 | creds, auth_url, base_path 43 | ) 44 | super().__init__( 45 | self.uftp_host, port=self.uftp_port, user="anonymous", passwd=self.uftp_password 46 | ) 47 | self.base_path = base_path 48 | 49 | def hassyspath(self, path): 50 | return False 51 | 52 | def validatepath(self, path): 53 | _p = "." + super().validatepath(path) 54 | print(f"validatepath: {path}->{_p}") 55 | return _p 56 | 57 | # work around for bug in FPTFS - _read_dir() should never be used 58 | def _read_dir(self, path): 59 | return OrderedDict({}) 60 | 61 | def __repr__(self): 62 | return f"UFTPFS({self.host!r}, port={self.port!r}), base_path={self.base_path!r}" 63 | 64 | __str__ = __repr__ 65 | 66 | 67 | class UFTPOpener(Opener): 68 | """Defines the Opener class used to open UFTP FS instances based on a URL""" 69 | 70 | protocols = ["uftp"] 71 | 72 | def _parse(self, resource_url): 73 | tok = resource_url.split("/rest/") 74 | auth_url = "https://" + tok[0] 75 | tok2 = tok[1].split(":") 76 | auth_url = auth_url + "/rest/" + tok2[0] 77 | base_dir = tok2[1] if len(tok) > 1 else "/" 78 | return auth_url, base_dir 79 | 80 | def _read_token(self, token_spec): 81 | token = None 82 | if token_spec: 83 | if token_spec.startswith("@@"): 84 | env_name = token_spec[2:] 85 | token = getenv(env_name, None) 86 | elif token_spec.startswith("@"): 87 | file_name = token_spec[1:] 88 | with open(file_name) as f: 89 | token = f.read().strip() 90 | else: 91 | token = token_spec 92 | return token 93 | 94 | def _create_credential(self, parse_result): 95 | token = self._read_token(parse_result.params.get("token", None)) 96 | return uc_credentials.create_credential( 97 | username=parse_result.username, 98 | password=parse_result.password, 99 | token=token, 100 | identity=parse_result.params.get("identity", None), 101 | ) 102 | 103 | def open_fs(self, fs_url, parse_result, writeable, create, cwd): 104 | auth_url, base_dir = self._parse(parse_result.resource) 105 | print("Base dir: ", base_dir) 106 | cred = self._create_credential(parse_result) 107 | return UFTPFS(auth_url, cred, base_dir) 108 | -------------------------------------------------------------------------------- /pyunicore/uftp/uftpfuse.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from errno import EIO 4 | from errno import ENOENT 5 | from errno import ENOSYS 6 | from errno import EROFS 7 | from time import time 8 | 9 | from fuse import FUSE 10 | from fuse import FuseOSError 11 | from fuse import Operations 12 | 13 | from pyunicore.uftp.uftp import UFTP 14 | 15 | 16 | class UFTPFile: 17 | """handle to an in-progress read or write transfer""" 18 | 19 | BUFFER_SIZE = 65536 20 | 21 | def __init__(self, path, uftp_session: UFTP): 22 | self.uftp_session = uftp_session 23 | self.path = path 24 | self.pos = 0 25 | self.data = None 26 | self.write_mode = False 27 | 28 | def close(self): 29 | self.close_data() 30 | 31 | def close_data(self): 32 | if self.data is not None: 33 | self.data.close() 34 | self.data = None 35 | self.pos = 0 36 | 37 | def open_data(self, position, write_mode=False): 38 | """open data channel before a sequence of read/write operations""" 39 | 40 | self.write_mode = write_mode 41 | if self.write_mode: 42 | _sock = self.uftp_session.get_write_socket(self.path, position) 43 | else: 44 | _sock = self.uftp_session.get_read_socket(self.path, position) 45 | self.pos = position 46 | if self.write_mode: 47 | self.data = _sock.makefile("wb", buffering=UFTPFile.BUFFER_SIZE) 48 | else: 49 | self.data = _sock.makefile("rb", buffering=UFTPFile.BUFFER_SIZE) 50 | 51 | def read(self, offset, size): 52 | if self.data is not None and self.write_mode: 53 | raise OSError("Not open for reading") 54 | if self.data is None: 55 | self.open_data(offset, write_mode=False) 56 | data_block = self.data.read(size) 57 | self.pos += len(data_block) 58 | return data_block 59 | 60 | def write(self, offset, data): 61 | if self.data is not None and not self.write_mode: 62 | raise OSError("Not open for writing") 63 | if self.data is None: 64 | self.open_data(offset, write_mode=True) 65 | to_write = len(data) 66 | write_offset = 0 67 | while to_write > 0: 68 | written = self.data.write(data[write_offset:]) 69 | if written is None: 70 | written = 0 71 | write_offset += written 72 | to_write -= written 73 | self.data.flush() 74 | self.pos += len(data) 75 | return len(data) 76 | 77 | 78 | class UFTPDriver(Operations): 79 | """ 80 | FUSE Driver 81 | Connects to uftpd at host:port with 82 | the given session password (which can be obtained by 83 | authenticating to an Authserver or UNICORE/X) 84 | """ 85 | 86 | def __init__(self, host, port, password, debug=False, read_only=False): 87 | self.host = host 88 | self.port = port 89 | self.password = password 90 | self.last_file = None 91 | self.file_map = {} 92 | self.next_file_handle = 0 93 | self.debug = debug 94 | self.read_only = read_only 95 | self.uftp_session = self.new_session() 96 | 97 | def new_session(self): 98 | uftp_session = UFTP() 99 | uftp_session.open_uftp_session(self.host, self.port, self.password) 100 | if self.debug: 101 | print("UFTP session initialised.") 102 | return uftp_session 103 | 104 | def chmod(self, path, mode): 105 | if self.debug: 106 | print(f"chmod {path} {mode}") 107 | if self.read_only: 108 | raise FuseOSError(EROFS) 109 | self.uftp_session.chmod(path, mode) 110 | 111 | def chown(self, path, uid, gid): 112 | raise FuseOSError(ENOSYS) 113 | 114 | def create(self, path, mode): 115 | if self.debug: 116 | print(f"create {path} {mode}") 117 | if self.read_only: 118 | raise FuseOSError(EROFS) 119 | fh = self.open(path, os.O_WRONLY) 120 | f = self.file_map[fh] 121 | f.write(0, []) 122 | # TBD f.chmod(mode) 123 | return fh 124 | 125 | def destroy(self, path): 126 | for f in self.file_map.values(): 127 | f.close() 128 | self.uftp_session.close() 129 | 130 | def getattr(self, path, fh=None): 131 | try: 132 | return self.uftp_session.stat(path) 133 | except OSError: 134 | raise FuseOSError(ENOENT) 135 | 136 | def mkdir(self, path, mode): 137 | if self.debug: 138 | print(f"mkdir {path} {mode}") 139 | if self.read_only: 140 | raise FuseOSError(EROFS) 141 | return self.uftp_session.mkdir(path, mode) 142 | 143 | def open(self, path, fi_flags): 144 | if self.read_only and (os.O_WRONLY & fi_flags): 145 | raise FuseOSError(EROFS) 146 | fh = self.next_file_handle 147 | if os.O_RDWR & fi_flags: 148 | raise FuseOSError(EIO) 149 | if self.debug: 150 | print(f"open [{fh}] {path} flags={fi_flags}") 151 | self.next_file_handle += 1 152 | f = UFTPFile(path, self.new_session()) 153 | self.file_map[fh] = f 154 | return fh 155 | 156 | def read(self, path, size, offset, fh): 157 | if self.debug: 158 | print(f"read [{fh}] {path} len={size} offset={offset}") 159 | f = self.file_map[fh] 160 | return f.read(offset, size) 161 | 162 | def readdir(self, path, fh): 163 | return [".", ".."] + [name for name in self.uftp_session.listdir(path)] 164 | 165 | def readlink(self, path): 166 | raise FuseOSError(ENOSYS) 167 | 168 | def rename(self, old, new): 169 | if self.read_only: 170 | raise FuseOSError(EROFS) 171 | return self.uftp_session.rename(old, new) 172 | 173 | def release(self, path, fh): 174 | if self.debug: 175 | print(f"release [{fh}] {path}") 176 | f = self.file_map[fh] 177 | f.close() 178 | self.file_map.__delitem__(fh) 179 | 180 | def rmdir(self, path): 181 | if self.read_only: 182 | raise FuseOSError(EROFS) 183 | if self.debug: 184 | print(f"rmdir {path}") 185 | return self.uftp_session.rmdir(path) 186 | 187 | def symlink(self, target, source): 188 | raise FuseOSError(ENOSYS) 189 | 190 | def truncate(self, path, length, fh=None): 191 | if self.read_only: 192 | raise FuseOSError(EROFS) 193 | if self.debug: 194 | print(f"truncate size={length} {path}") 195 | pass 196 | 197 | def unlink(self, path): 198 | if self.read_only: 199 | raise FuseOSError(EROFS) 200 | return self.uftp_session.rm(path) 201 | 202 | def utimens(self, path, times=None): 203 | if self.read_only: 204 | raise FuseOSError(EROFS) 205 | if times is None: 206 | _time = time() 207 | else: 208 | _time = times[0] 209 | self.uftp_session.set_time(path, _time) 210 | 211 | def write(self, path, data, offset, fh): 212 | if self.debug: 213 | print(f"write [{fh}] {path} len={len(data)} offset={offset}") 214 | f = self.file_map[fh] 215 | f.write(offset, data) 216 | return len(data) 217 | 218 | 219 | def main(): 220 | parser = argparse.ArgumentParser() 221 | parser.add_argument( 222 | "-d", 223 | "--debug", 224 | action="store_true", 225 | help="debug mode (also keeps process in the foreground)", 226 | ) 227 | parser.add_argument( 228 | "-r", 229 | "--read-only", 230 | action="store_true", 231 | help="read-only mode, prevents writes, renames, etc", 232 | ) 233 | parser.add_argument( 234 | "-f", 235 | "--foreground", 236 | action="store_true", 237 | help="run fusedriver in foreground", 238 | ) 239 | parser.add_argument( 240 | "-P", 241 | "--password", 242 | help="one-time password (if not given, it is expected in the environment UFTP_PASSWORD)", 243 | ) 244 | parser.add_argument( 245 | "-o", 246 | "--fuse-options", 247 | help="additional options (key1=value1,key2=value2,...) for fusepy", 248 | ) 249 | parser.add_argument("address", help="UFTPD server's address (host:port)") 250 | parser.add_argument( 251 | "mount_point", 252 | help="the local mount directory (must exist and be empty)", 253 | ) 254 | 255 | args = parser.parse_args() 256 | _pwd = args.password 257 | if _pwd is None: 258 | _pwd = os.getenv("UFTP_PASSWORD") 259 | if _pwd is None: 260 | raise TypeError( 261 | "UFTP one-time password must be given with '-P ...' or as environment UFTP_PASSWORD" 262 | ) 263 | _host, _port = args.address.split(":") 264 | foreground = args.foreground or args.debug 265 | extra_opts = _parse_args(args.fuse_options) 266 | driver = UFTPDriver(_host, int(_port), _pwd, debug=args.debug, read_only=args.read_only) 267 | FUSE( 268 | driver, 269 | args.mount_point, 270 | debug=args.debug, 271 | foreground=foreground, 272 | nothreads=True, 273 | **extra_opts, 274 | ) 275 | 276 | 277 | def _parse_args(args: str) -> dict: 278 | result = {} 279 | if args: 280 | for opt in args.split(","): 281 | kv = opt.split("=") if "=" in opt else [opt, True] 282 | result[kv[0]] = kv[1] 283 | return result 284 | 285 | 286 | if __name__ == "__main__": 287 | main() 288 | -------------------------------------------------------------------------------- /pyunicore/uftp/uftpmountfs.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from os import getenv 3 | 4 | from fs.opener import Opener 5 | from fs.osfs import OSFS 6 | 7 | import pyunicore.credentials as uc_credentials 8 | from pyunicore.uftp.uftp import UFTP 9 | 10 | 11 | class UFTPMOUNTFS(OSFS): 12 | """A UFTP (UNICORE FTP) Filesystem which tries to mount the remote file system 13 | and then access it locally. 14 | 15 | This extends the fs.osfs.OSFS filesystem 16 | with UFTP authentication. 17 | 18 | Example: create with auth URL and username/password credentials 19 | 20 | from pyunicore.credentials import UsernamePassword 21 | from pyunicore.uftp.uftpmountfs import UFTPMOUNTFS 22 | 23 | auth = "https://localhost:9000/rest/auth/TEST" 24 | creds = UsernamePassword("demouser", "test123") 25 | base_path = "/" 26 | mount_dir = "/data/mount" 27 | 28 | uftp_mount_fs = UFTPMOUNTFS(auth, creds, base_path, mount_dir) 29 | uftp_mount_fs.tree() 30 | 31 | Example: create with URL 32 | 33 | from fs import open_fs 34 | fs_url = "uftpmount://demouser:test123@localhost:9000/rest/auth/TEST:/opt/shared-data==/mnt" 35 | uftp_fs = open_fs(fs_url) 36 | uftp_fs.tree() 37 | """ 38 | 39 | def __init__(self, auth_url, creds, base_path="/", mount_dir="./uftp_mount"): 40 | """Creates a new UFTP FS instance authenticating using the given URL and credentials""" 41 | self.host, self.port, self.uftp_password = UFTP().authenticate(creds, auth_url, base_path) 42 | self.base_path = base_path 43 | self.mount_dir = mount_dir 44 | self._ensure_unmount() 45 | self._run_fusedriver(self.uftp_password) 46 | super().__init__(mount_dir) 47 | 48 | def close(self): 49 | self._ensure_unmount() 50 | super().close() 51 | 52 | def _ensure_unmount(self): 53 | """ 54 | Unmounts the requested directory 55 | """ 56 | cmd = "fusermount -u '%s'" % self.mount_dir 57 | return self._run_command(cmd) 58 | 59 | def _run_fusedriver(self, pwd): 60 | cmds = [ 61 | "export UFTP_PASSWORD=%s" % pwd, 62 | f"python3 -m pyunicore.uftp.uftpfuse {self.host}:{self.port} '{self.mount_dir}'", 63 | ] 64 | cmd = "" 65 | for c in cmds: 66 | cmd += c + "\n" 67 | return self._run_command(cmd) 68 | 69 | def _run_command(self, cmd): 70 | try: 71 | raw_output = subprocess.check_output( 72 | cmd, shell=True, bufsize=4096, stderr=subprocess.STDOUT 73 | ) 74 | exit_code = 0 75 | except subprocess.CalledProcessError as cpe: 76 | raw_output = cpe.output 77 | exit_code = cpe.returncode 78 | return exit_code, raw_output.decode("UTF-8") 79 | 80 | def hassyspath(self, _): 81 | return False 82 | 83 | def __repr__(self): 84 | return ( 85 | f"UFTPMOUNTFS({self.host!r}, port={self.port!r})" 86 | ", base_path={self.base_path!r}, mount_dir={self.mount_dir!r}" 87 | ) 88 | 89 | __str__ = __repr__ 90 | 91 | 92 | class UFTPMountOpener(Opener): 93 | """Defines the Opener class used to open UFTP Mount FS instances based on a URL""" 94 | 95 | protocols = ["uftpmount"] 96 | 97 | def _parse(self, resource_url): 98 | tok = resource_url.split("/rest/") 99 | auth_url = "https://" + tok[0] 100 | tok2 = tok[1].split(":") 101 | auth_url = auth_url + "/rest/" + tok2[0] 102 | base_dir = tok2[1] if len(tok) > 1 else "/" 103 | tok3 = base_dir.split("==") 104 | base_dir = tok3[0] 105 | mount_dir = tok3[1] if len(tok3) > 1 else "/uftp_mount" 106 | return auth_url, base_dir, mount_dir 107 | 108 | def _read_token(self, token_spec): 109 | token = None 110 | if token_spec: 111 | if token_spec.startswith("@@"): 112 | env_name = token_spec[2:] 113 | token = getenv(env_name, None) 114 | elif token_spec.startswith("@"): 115 | file_name = token_spec[1:] 116 | with open(file_name) as f: 117 | token = f.read().strip() 118 | else: 119 | token = token_spec 120 | return token 121 | 122 | def _create_credential(self, parse_result): 123 | token = self._read_token(parse_result.params.get("token", None)) 124 | return uc_credentials.create_credential( 125 | username=parse_result.username, 126 | password=parse_result.password, 127 | token=token, 128 | identity=parse_result.params.get("identity", None), 129 | ) 130 | 131 | def open_fs(self, fs_url, parse_result, writeable, create, cwd): 132 | auth_url, base_dir, mount_dir = self._parse(parse_result.resource) 133 | cred = self._create_credential(parse_result) 134 | return UFTPMOUNTFS(auth_url, cred, base_dir, mount_dir) 135 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | pre-commit 4 | fs 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests>=2.5.1 2 | PyJWT>=2.0 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [versioneer] 2 | VCS = git 3 | style = pep440 4 | versionfile_source = pyunicore/_version.py 5 | versionfile_build = pyunicore/_version.py 6 | tag_prefix = 7 | parentdir_prefix = 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import find_packages 3 | from setuptools import setup 4 | 5 | import versioneer 6 | 7 | long_description = """ 8 | This library provides a Python wrapper for the UNICORE REST API, making common tasks like file access, job submission and management, workflow submission and management more convenient, and integrating UNICORE features better with typical Python usage. 9 | 10 | Visit https://github.com/HumanBrainProject/pyunicore for more information. 11 | """ 12 | 13 | python_requires = ">=3.7" 14 | 15 | install_requires = [ 16 | "pyjwt>=2.8", 17 | "requests>=2.5", 18 | ] 19 | 20 | extras_require = { 21 | "fuse": ["fusepy>=3.0.1"], 22 | "crypto": ["cryptography>=3.3.1", "bcrypt>=4.0.0"], 23 | "fs": ["fs>=2.4.0"], 24 | } 25 | 26 | setup( 27 | name="pyunicore", 28 | version=versioneer.get_version(), 29 | cmdclass=versioneer.get_cmdclass(), 30 | packages=find_packages(), 31 | author="Bernd Schuller", 32 | author_email="b.schuller@fz-juelich.de", 33 | description="Python library for using the UNICORE REST API", 34 | long_description=long_description, 35 | python_requires=python_requires, 36 | install_requires=install_requires, 37 | extras_require=extras_require, 38 | entry_points={ 39 | "fs.opener": [ 40 | "uftp = pyunicore.uftp.uftpfs:UFTPOpener", 41 | "uftpmount = pyunicore.uftp.uftpmountfs:UFTPMountOpener", 42 | ], 43 | "console_scripts": [ 44 | "unicore-port-forwarder=pyunicore.forwarder:main", 45 | "unicore-cwl-runner=pyunicore.cwl.cwltool:main", 46 | "unicore-fusedriver=pyunicore.uftp.uftpfuse:main", 47 | "unicore=pyunicore.cli.main:main", 48 | ], 49 | }, 50 | license="License :: OSI Approved :: BSD", 51 | url="https://github.com/HumanBrainProject/pyunicore", 52 | ) 53 | -------------------------------------------------------------------------------- /tests/cwldocs/array-inputs.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwltool 2 | 3 | cwlVersion: v1.0 4 | class: CommandLineTool 5 | inputs: 6 | filesA: 7 | type: string[] 8 | inputBinding: 9 | prefix: -A 10 | position: 1 11 | 12 | filesB: 13 | type: 14 | type: array 15 | items: string 16 | inputBinding: 17 | prefix: -B= 18 | separate: false 19 | inputBinding: 20 | position: 2 21 | 22 | filesC: 23 | type: string[] 24 | inputBinding: 25 | prefix: -C= 26 | itemSeparator: "," 27 | separate: false 28 | position: 3 29 | 30 | outputs: 31 | example_out: 32 | type: stdout 33 | stdout: output.txt 34 | baseCommand: echo 35 | -------------------------------------------------------------------------------- /tests/cwldocs/array-inputs.params: -------------------------------------------------------------------------------- 1 | filesA: [ one, two, three ] 2 | filesB: [ four, five, six ] 3 | filesC: [ seven, eight, nine ] 4 | -------------------------------------------------------------------------------- /tests/cwldocs/directoryinput.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwltool 2 | 3 | cwlVersion: v1.0 4 | class: CommandLineTool 5 | baseCommand: test_command 6 | inputs: 7 | file_1: 8 | type: Directory 9 | inputBinding: 10 | prefix: --input= 11 | separate: false 12 | position: 1 13 | 14 | outputs: [] 15 | -------------------------------------------------------------------------------- /tests/cwldocs/directoryinput.params: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "file_1": { 4 | "class": "File", 5 | "path": "tests/cwldocs" 6 | }, 7 | 8 | } 9 | -------------------------------------------------------------------------------- /tests/cwldocs/echo.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwltool 2 | 3 | cwlVersion: v1.0 4 | class: CommandLineTool 5 | baseCommand: echo 6 | inputs: 7 | param_1: 8 | type: string 9 | inputBinding: 10 | position: 1 11 | param_2: 12 | type: string 13 | inputBinding: 14 | position: 2 15 | param_3: 16 | type: int 17 | inputBinding: 18 | position: 3 19 | param_4: 20 | type: int[] 21 | inputBinding: 22 | itemSeparator: "," 23 | position: 4 24 | param_5: 25 | type: 26 | type: array 27 | items: int 28 | inputBinding: 29 | prefix: "-x" 30 | inputBinding: 31 | position: 5 32 | 33 | outputs: [] 34 | 35 | stdout: my_out 36 | stderr: my_err 37 | -------------------------------------------------------------------------------- /tests/cwldocs/echo.params: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "param_1": "hello world!", 4 | 5 | "param_2": "two", 6 | 7 | "param_3": 42, 8 | 9 | "param_4": [1, 2, 3], 10 | 11 | "param_5": [7, 8] 12 | 13 | } 14 | -------------------------------------------------------------------------------- /tests/cwldocs/fetching_data_tool.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwltool 2 | 3 | cwlVersion: v1.0 4 | class: CommandLineTool 5 | baseCommand: echo 6 | hints: 7 | DockerRequirement: 8 | dockerPull: docker-registry.ebrains.eu/tc/cwl-workflows/psd_workflow_fetching_data:latest 9 | ResourceRequirement: 10 | ramMin: 2048 11 | outdirMin: 4096 12 | inputs: 13 | bucket_id: 14 | type: string 15 | inputBinding: 16 | position: 1 17 | object_name: 18 | type: string 19 | inputBinding: 20 | position: 2 21 | token: 22 | type: string 23 | inputBinding: 24 | position: 3 25 | outputs: 26 | fetched_file: 27 | type: File 28 | outputBinding: 29 | glob: $(inputs.object_name) 30 | -------------------------------------------------------------------------------- /tests/cwldocs/fetching_data_tool.params: -------------------------------------------------------------------------------- 1 | bucket_id: 123 2 | 3 | token: some_token 4 | 5 | object_name: my_container 6 | -------------------------------------------------------------------------------- /tests/cwldocs/fileinput.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwltool 2 | 3 | cwlVersion: v1.0 4 | class: CommandLineTool 5 | baseCommand: test_command 6 | inputs: 7 | file_1: 8 | type: File 9 | inputBinding: 10 | prefix: --file1= 11 | separate: false 12 | position: 1 13 | file_2: 14 | type: File 15 | inputBinding: 16 | prefix: --file2 17 | separate: true 18 | position: 2 19 | 20 | outputs: [] 21 | -------------------------------------------------------------------------------- /tests/cwldocs/fileinput.params: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "file_1": { 4 | "class": "File", 5 | "path": "test.sh" 6 | }, 7 | 8 | "file_2": { 9 | "class": "File", 10 | "path": "file2" 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /tests/cwldocs/fileinput_remote.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwltool 2 | 3 | cwlVersion: v1.0 4 | class: CommandLineTool 5 | baseCommand: test_command 6 | inputs: 7 | file_1: 8 | type: File 9 | inputBinding: 10 | prefix: --file1= 11 | separate: false 12 | position: 1 13 | file_2: 14 | type: File 15 | inputBinding: 16 | prefix: --file2 17 | separate: true 18 | position: 2 19 | file_3: 20 | type: File 21 | inputBinding: 22 | position: 3 23 | file_4: 24 | type: File 25 | inputBinding: 26 | position: 4 27 | 28 | outputs: [] 29 | -------------------------------------------------------------------------------- /tests/cwldocs/fileinput_remote.params: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "file_1": { 4 | "class": "File", 5 | "path": "test.sh" 6 | }, 7 | 8 | "file_2": { 9 | "class": "File", 10 | "path": "file2" 11 | }, 12 | 13 | "file_3": { 14 | "class": "File", 15 | "location": "https://www.google.de", 16 | "basename": "some_remote_file" 17 | }, 18 | 19 | "file_4": { 20 | "class": "File", 21 | "location": "https://www.foo.bar/file.txt", 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /tests/integration/cli/jobs/date.u: -------------------------------------------------------------------------------- 1 | { 2 | "ApplicationName": "Date" 3 | } 4 | -------------------------------------------------------------------------------- /tests/integration/cli/preferences: -------------------------------------------------------------------------------- 1 | # User preferences for UCC 2 | 3 | authentication-method=USERNAME 4 | username=demouser 5 | password=test123 6 | 7 | verbose=true 8 | 9 | # 10 | # The address(es) of the registries to contact 11 | # (space separated list) 12 | registry=https://localhost:8080/DEMO-SITE/rest/registries/default_registry 13 | contact-registry=true 14 | 15 | # 16 | # default directory for output 17 | # 18 | output=/tmp 19 | -------------------------------------------------------------------------------- /tests/integration/cli/test_base.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.cli.base as base 4 | 5 | 6 | class TestBase(unittest.TestCase): 7 | 8 | def test_base_setup(self): 9 | cmd = base.Base() 10 | cmd.config_file = "tests/integration/cli/preferences" 11 | cmd.load_user_properties() 12 | registry = cmd.create_registry() 13 | self.assertTrue(len(registry.site_urls) > 0) 14 | print(registry.site_urls) 15 | 16 | def test_issue_token(self): 17 | cmd = base.IssueToken() 18 | config_file = "tests/integration/cli/preferences" 19 | ep = "https://localhost:8080/DEMO-SITE/rest/core" 20 | args = ["-c", config_file, ep, "--lifetime", "700", "--inspect", "--limited", "--renewable"] 21 | cmd.run(args) 22 | 23 | def test_rest_get(self): 24 | cmd = base.REST() 25 | config_file = "tests/integration/cli/preferences" 26 | ep = "https://localhost:8080/DEMO-SITE/rest/core" 27 | args = ["-c", config_file, "GET", ep] 28 | cmd.run(args) 29 | 30 | def test_rest_post_put_delete(self): 31 | cmd = base.REST() 32 | config_file = "tests/integration/cli/preferences" 33 | ep = "https://localhost:8080/DEMO-SITE/rest/core/sites" 34 | d = "{}" 35 | args = ["-c", config_file, "--data", d, "POST", ep] 36 | cmd.run(args) 37 | d = "{'tags': 'test123'}" 38 | ep = cmd._last_location 39 | args = ["-c", config_file, "--data", d, "PUT", ep] 40 | cmd.run(args) 41 | args = ["-c", config_file, "DELETE", ep] 42 | cmd.run(args) 43 | 44 | 45 | if __name__ == "__main__": 46 | unittest.main() 47 | -------------------------------------------------------------------------------- /tests/integration/cli/test_exec.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.cli.exec as exec 4 | 5 | 6 | class TestExec(unittest.TestCase): 7 | 8 | def test_exec(self): 9 | cmd = exec.Exec() 10 | config_file = "tests/integration/cli/preferences" 11 | ep = "https://localhost:8080/DEMO-SITE/rest/core" 12 | args = ["-c", config_file, "-v", "--keep", "--server-url", ep, "date"] 13 | cmd.run(args) 14 | 15 | def test_run_1(self): 16 | cmd = exec.Run() 17 | config_file = "tests/integration/cli/preferences" 18 | ep = "https://localhost:8080/DEMO-SITE/rest/core" 19 | jobfile = "tests/integration/cli/jobs/date.u" 20 | args = ["-c", config_file, "-v", "--server-url", ep, jobfile] 21 | cmd.run(args) 22 | 23 | def test_list_jobs(self): 24 | cmd = exec.ListJobs() 25 | config_file = "tests/integration/cli/preferences" 26 | args = ["-c", config_file, "-v", "-l"] 27 | cmd.run(args) 28 | 29 | 30 | if __name__ == "__main__": 31 | unittest.main() 32 | -------------------------------------------------------------------------------- /tests/integration/cli/test_io.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.cli.base as base 4 | import pyunicore.cli.io as io 5 | import pyunicore.client 6 | 7 | 8 | class TestIO(unittest.TestCase): 9 | 10 | def test_crawl(self): 11 | cmd = base.Base() 12 | cmd.config_file = "tests/integration/cli/preferences" 13 | cmd.load_user_properties() 14 | ep = "https://localhost:8080/DEMO-SITE/rest/core/storages/HOME" 15 | registry = cmd.create_registry() 16 | self.assertTrue(len(registry.site_urls) > 0) 17 | storage = pyunicore.client.Storage(registry.transport, ep) 18 | for x in io.crawl_remote(storage, "/", "*"): 19 | print(x) 20 | 21 | def test_ls(self): 22 | cmd = io.LS() 23 | config_file = "tests/integration/cli/preferences" 24 | ep = "https://localhost:8080/DEMO-SITE/rest/core/storages/HOME" 25 | args = ["-c", config_file, "-v", "--long", ep] 26 | cmd.run(args) 27 | 28 | 29 | if __name__ == "__main__": 30 | unittest.main() 31 | -------------------------------------------------------------------------------- /tests/integration/files/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | date 4 | -------------------------------------------------------------------------------- /tests/integration/files/workflow1.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "variables": [ 4 | { 5 | "id": "decl1", 6 | "name": "COUNTER", 7 | "type": "INTEGER", 8 | "initial_value": "123" 9 | } 10 | ], 11 | 12 | "activities" : [ 13 | 14 | { 15 | "id": "date1", 16 | "job": { 17 | "Executable": "date", 18 | "Exports": [ 19 | {"From": "stdout", "To": "wf:date1/stdout"} 20 | ] 21 | } 22 | }, 23 | 24 | { 25 | "id": "hold1", 26 | "type": "HOLD" 27 | }, 28 | 29 | { 30 | "id": "date2", 31 | "job": { 32 | "Executable": "date", 33 | "Exports": [ 34 | {"From": "stdout", "To": "wf:date2/stdout"} 35 | ] 36 | } 37 | } 38 | ], 39 | 40 | "transitions": [ 41 | { "from": "date1", "to": "hold1" }, 42 | { "from": "hold1", "to": "date2" } 43 | 44 | ] 45 | 46 | } 47 | -------------------------------------------------------------------------------- /tests/integration/test_auth.py: -------------------------------------------------------------------------------- 1 | import json 2 | import unittest 3 | 4 | import pyunicore.client as uc_client 5 | import pyunicore.credentials as uc_credentials 6 | 7 | 8 | class TestAuth(unittest.TestCase): 9 | def setUp(self): 10 | pass 11 | 12 | def get_client(self, credential=None): 13 | if credential is None: 14 | credential = uc_credentials.UsernamePassword("demouser", "test123") 15 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 16 | transport = uc_client.Transport(credential) 17 | return uc_client.Client(transport, base_url) 18 | 19 | def test_username_auth(self): 20 | print("*** test_username_auth") 21 | client = self.get_client() 22 | print(json.dumps(client.access_info(), indent=2)) 23 | self.assertEqual("user", client.properties["client"]["role"]["selected"]) 24 | 25 | def test_transport_settings(self): 26 | print("*** test_transport_settings") 27 | client = self.get_client() 28 | ai = client.access_info() 29 | print(json.dumps(ai, indent=2)) 30 | grps = ai["xlogin"]["availableGroups"] 31 | for grp in grps: 32 | client.transport.preferences = f"group:{grp}" 33 | ai = client.access_info() 34 | print("Selected group:", ai["xlogin"]["group"]) 35 | self.assertEqual(grp, ai["xlogin"]["group"]) 36 | 37 | def test_anonymous_info(self): 38 | print("*** test_anonymous_info") 39 | cred = uc_credentials.Anonymous() 40 | client = self.get_client(cred) 41 | self.assertEqual("anonymous", client.properties["client"]["role"]["selected"]) 42 | 43 | def test_issue_auth_token(self): 44 | print("*** test_issue_auth_token") 45 | client = self.get_client() 46 | if client.server_version_info() < (9, 2, 0): 47 | print("Skipping, requires server 9.2.0 or later") 48 | return 49 | token = client.issue_auth_token(lifetime=600, limited=True) 50 | print("token: %s" % token) 51 | 52 | 53 | if __name__ == "__main__": 54 | unittest.main() 55 | -------------------------------------------------------------------------------- /tests/integration/test_basic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | import pyunicore.client as uc_client 5 | import pyunicore.credentials as uc_credentials 6 | 7 | 8 | class TestBasic(unittest.TestCase): 9 | def setUp(self): 10 | pass 11 | 12 | def get_client(self, credential=None): 13 | if credential is None: 14 | credential = uc_credentials.UsernamePassword("demouser", "test123") 15 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 16 | transport = uc_client.Transport(credential) 17 | return uc_client.Client(transport, base_url) 18 | 19 | def test_connect(self): 20 | print("*** test_connect") 21 | client = self.get_client() 22 | self.assertEqual("user", client.properties["client"]["role"]["selected"]) 23 | 24 | def test_run_date(self): 25 | print("*** test_run_date") 26 | client = self.get_client() 27 | job_desc = {"Executable": "date"} 28 | job = client.new_job(job_desc) 29 | print(job) 30 | job.cache_time = 0 31 | job.poll() 32 | exit_code = int(job.properties["exitCode"]) 33 | self.assertEqual(0, exit_code) 34 | work_dir = job.working_dir 35 | stdout = work_dir.stat("/stdout").raw().read() 36 | self.assertTrue(len(stdout) > 0) 37 | print(stdout) 38 | 39 | def test_run_uploaded_script(self): 40 | print("*** test_run_uploaded_script") 41 | client = self.get_client() 42 | job_desc = {"Executable": "bash", "Arguments": ["script.sh"]} 43 | in_file = os.getcwd() + "/tests/integration/files/script.sh" 44 | job = client.new_job(job_desc, [in_file]) 45 | job.poll() 46 | exit_code = int(job.properties["exitCode"]) 47 | self.assertEqual(0, exit_code) 48 | work_dir = job.working_dir 49 | stdout = work_dir.stat("/stdout").raw().read() 50 | self.assertTrue(len(stdout) > 0) 51 | print(stdout) 52 | 53 | def test_run_uploaded_script_2(self): 54 | print("*** test_run_uploaded_script_2") 55 | client = self.get_client() 56 | job_desc = {"Executable": "bash", "Arguments": ["myscript.sh"]} 57 | in_file = os.getcwd() + "/tests/integration/files/script.sh" 58 | job = client.new_job(job_desc, {"myscript.sh": in_file}) 59 | job.poll() 60 | exit_code = int(job.properties["exitCode"]) 61 | self.assertEqual(0, exit_code) 62 | work_dir = job.working_dir 63 | stdout = work_dir.stat("/stdout").raw().read() 64 | self.assertTrue(len(stdout) > 0) 65 | print(stdout) 66 | 67 | def test_alloc_and_run_date(self): 68 | print("*** test_alloc_and_run_date") 69 | client = self.get_client() 70 | if client.server_version_info() < (9, 0, 0): 71 | print("Skipping, requires server 9.0.0 or later") 72 | return 73 | alloc_desc = {"Job type": "ALLOCATE", "Resources": {"Runtime": "10m"}} 74 | allocation = client.new_job(alloc_desc) 75 | try: 76 | print(allocation) 77 | allocation.wait_until_available() 78 | if allocation.status != uc_client.JobStatus.RUNNING: 79 | print("Skipping, allocation not available.") 80 | return 81 | job_desc = {"Executable": "date"} 82 | job = allocation.new_job(job_desc) 83 | print(job) 84 | job.cache_time = 0 85 | job.poll() 86 | exit_code = int(job.properties["exitCode"]) 87 | self.assertEqual(0, exit_code) 88 | work_dir = job.working_dir 89 | stdout = work_dir.stat("/stdout").raw().read() 90 | self.assertTrue(len(stdout) > 0) 91 | print(stdout) 92 | finally: 93 | allocation.abort() 94 | 95 | 96 | if __name__ == "__main__": 97 | unittest.main() 98 | -------------------------------------------------------------------------------- /tests/integration/test_registry.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.client as uc_client 4 | import pyunicore.credentials as uc_credentials 5 | 6 | 7 | class TestRegistry(unittest.TestCase): 8 | def setUp(self): 9 | pass 10 | 11 | def get_registry(self): 12 | credential = uc_credentials.UsernamePassword("demouser", "test123") 13 | base_url = "https://localhost:8080/DEMO-SITE/rest/registries/default_registry" 14 | transport = uc_client.Transport(credential) 15 | return uc_client.Registry(transport, base_url) 16 | 17 | def test_connect(self): 18 | print("*** test_connect") 19 | registry = self.get_registry() 20 | print("Registry contains: ", registry.site_urls) 21 | site_client = registry.site("DEMO-SITE") 22 | self.assertEqual("user", site_client.properties["client"]["role"]["selected"]) 23 | workflow_client = registry.workflow_service("DEMO-SITE") 24 | self.assertEqual("user", workflow_client.properties["client"]["role"]["selected"]) 25 | 26 | 27 | if __name__ == "__main__": 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /tests/integration/test_storage.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from io import BytesIO 4 | from time import sleep 5 | 6 | import pyunicore.client as uc_client 7 | import pyunicore.credentials as uc_credentials 8 | 9 | 10 | class TestBasic(unittest.TestCase): 11 | def setUp(self): 12 | pass 13 | 14 | def get_client(self): 15 | credential = uc_credentials.UsernamePassword("demouser", "test123") 16 | base_url = "https://localhost:8080/DEMO-SITE/rest/core" 17 | transport = uc_client.Transport(credential) 18 | return uc_client.Client(transport, base_url) 19 | 20 | def get_home_storage(self): 21 | credential = uc_credentials.UsernamePassword("demouser", "test123") 22 | transport = uc_client.Transport(credential) 23 | return uc_client.Storage( 24 | transport, 25 | "https://localhost:8080/DEMO-SITE/rest/core/storages/HOME", 26 | ) 27 | 28 | def test_list_storages(self): 29 | print("*** test_list_storages") 30 | site_client = self.get_client() 31 | storages = site_client.get_storages() 32 | home = None 33 | for s in storages: 34 | print(s) 35 | if "storages/HOME" in s.resource_url: 36 | home = s 37 | break 38 | self.assertIsNotNone(home) 39 | home.listdir() 40 | home.listdir(".") 41 | home.listdir("/") 42 | 43 | def test_upload_download(self): 44 | print("*** test_upload_download") 45 | home = self.get_home_storage() 46 | _path = "tests/integration/files/script.sh" 47 | _length = os.stat(_path).st_size 48 | with open(_path, "rb") as f: 49 | home.put(f, "script.sh") 50 | remote_file = home.stat("script.sh") 51 | self.assertEqual(_length, int(remote_file.properties["size"])) 52 | _out = BytesIO() 53 | remote_file.download(_out) 54 | self.assertEqual(_length, len(str(_out.getvalue(), "UTF-8"))) 55 | 56 | def test_upload_download_data(self): 57 | print("*** test_upload_download_data") 58 | home = self.get_home_storage() 59 | _data = "this is some test data" 60 | _length = len(_data) 61 | home.put(_data, "test.txt") 62 | remote_file = home.stat("test.txt") 63 | self.assertEqual(_length, int(remote_file.properties["size"])) 64 | _out = BytesIO() 65 | remote_file.download(_out) 66 | self.assertEqual(_length, len(str(_out.getvalue(), "UTF-8"))) 67 | 68 | def test_transfer(self): 69 | print("*** test_transfer") 70 | storage1 = self.get_home_storage() 71 | _path = "tests/integration/files/script.sh" 72 | _length = os.stat(_path).st_size 73 | with open(_path, "rb") as f: 74 | storage1.put(f, "script.sh") 75 | site_client = self.get_client() 76 | storage2 = site_client.new_job({}).working_dir 77 | transfer = storage2.receive_file(storage1.resource_url + "/files/script.sh", "script.sh") 78 | print(transfer) 79 | while transfer.is_running(): 80 | sleep(2) 81 | print("Transferred bytes: %s" % transfer.properties["transferredBytes"]) 82 | self.assertEqual(_length, int(transfer.properties["transferredBytes"])) 83 | transfer2 = storage1.send_file("script.sh", storage2.resource_url + "/files/script2.sh") 84 | print(transfer2) 85 | transfer2.poll() 86 | print("Transferred bytes: %s" % transfer2.properties["transferredBytes"]) 87 | self.assertEqual(_length, int(transfer2.properties["transferredBytes"])) 88 | for t in site_client.get_transfers(): 89 | print(t) 90 | 91 | 92 | if __name__ == "__main__": 93 | unittest.main() 94 | -------------------------------------------------------------------------------- /tests/integration/test_workflow.py: -------------------------------------------------------------------------------- 1 | import json 2 | import unittest 3 | from time import sleep 4 | 5 | import pyunicore.client as uc_client 6 | import pyunicore.credentials as uc_credentials 7 | 8 | 9 | class TestBasic(unittest.TestCase): 10 | def setUp(self): 11 | pass 12 | 13 | def get_client(self): 14 | credential = uc_credentials.UsernamePassword("demouser", "test123") 15 | base_url = "https://localhost:8080/DEMO-SITE/rest/workflows" 16 | transport = uc_client.Transport(credential) 17 | return uc_client.WorkflowService(transport, base_url) 18 | 19 | def test_run_workflow(self): 20 | print("*** test_run_workflow") 21 | wf_service = self.get_client() 22 | with open("tests/integration/files/workflow1.json") as _f: 23 | wf = json.load(_f) 24 | workflow1 = wf_service.new_workflow(wf) 25 | print("Submitted %s" % workflow1.resource_url) 26 | print("... waiting for workflow to go into HELD state") 27 | while not workflow1.is_held(): 28 | sleep(2) 29 | params = workflow1.properties["parameters"] 30 | print("... workflow variables: %s" % params) 31 | params["COUNTER"] = "789" 32 | print("... resuming workflow with params = %s" % params) 33 | workflow1.resume(params) 34 | print("... waiting for workflow to finish") 35 | workflow1.poll() 36 | params = workflow1.properties["parameters"] 37 | print("Final workflow variables: %s" % params) 38 | self.assertEqual("789", params["COUNTER"]) 39 | self.assertEqual(2, len(workflow1.get_files())) 40 | self.assertEqual(2, len(workflow1.get_jobs())) 41 | print("Output from date1: %s " % workflow1.stat("wf:date1/stdout").raw().read()) 42 | print("Output from date2: %s " % workflow1.stat("wf:date2/stdout").raw().read()) 43 | 44 | 45 | if __name__ == "__main__": 46 | unittest.main() 47 | -------------------------------------------------------------------------------- /tests/testing/__init__.py: -------------------------------------------------------------------------------- 1 | from tests.testing.contexts import expect_raise_if_exception 2 | from tests.testing.pyunicore import FakeClient 3 | from tests.testing.pyunicore import FakeJob 4 | from tests.testing.pyunicore import FakeRegistry 5 | from tests.testing.pyunicore import FakeTransport 6 | -------------------------------------------------------------------------------- /tests/testing/contexts.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | from typing import Any 3 | 4 | import pytest # type: ignore 5 | 6 | 7 | def expect_raise_if_exception( 8 | expected: Any, 9 | ): 10 | """Create a context that expects a raised exception or no raised exception. 11 | 12 | Args: 13 | expected: The expected result. 14 | 15 | Returns: 16 | _pytest.python_api.RaisesContext: If expected is of type `Exception` 17 | contextlib.suppress: otherwise. 18 | 19 | """ 20 | return ( 21 | pytest.raises(type(expected)) if isinstance(expected, Exception) else contextlib.suppress() 22 | ) 23 | -------------------------------------------------------------------------------- /tests/testing/pyunicore.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | from typing import List 3 | 4 | 5 | class FakeTransport: 6 | def __init__(self, auth_token: str = "test_token", oidc: bool = True): 7 | self.auth_token = auth_token 8 | self.oidc = oidc 9 | 10 | def _clone(self) -> "FakeTransport": 11 | return self 12 | 13 | def get(self, url): 14 | return {"entries": [{"href": "test-entry-url", "type": "test-entry-type"}]} 15 | 16 | 17 | class FakeRegistry: 18 | def __init__( 19 | self, 20 | transport: FakeTransport, 21 | url: str = "test_registry_url", 22 | contains: Dict[str, str] = None, 23 | ): 24 | self.transport = transport 25 | self.url = url 26 | self.site_urls = contains or {} 27 | 28 | @property 29 | def properties(self) -> dict: 30 | return self.__dict__ 31 | 32 | 33 | class FakeJob: 34 | def __init__( 35 | self, 36 | transport: FakeTransport, 37 | job_url: str = "test-job", 38 | properties: Dict = None, 39 | existing_files: Dict[str, str] = None, 40 | will_be_successful: bool = True, 41 | ): 42 | self.transport = transport 43 | self.url = job_url 44 | self._properties = properties or {"status": "QUEUED"} 45 | self._existing_files = existing_files or {} 46 | self._successful = will_be_successful 47 | 48 | @property 49 | def properties(self) -> Dict: 50 | return self._properties 51 | 52 | @property 53 | def job_id(self) -> str: 54 | return self.url 55 | 56 | def poll(self) -> None: 57 | self._properties["status"] = "SUCCESSFUL" if self._successful else "FAILED" 58 | 59 | def abort(self): 60 | pass 61 | 62 | 63 | class FakeClient: 64 | def __init__( 65 | self, 66 | transport: FakeTransport = None, 67 | site_url: str = "test_api_url", 68 | login_successful: bool = False, 69 | ): 70 | if transport is None: 71 | transport = FakeTransport() 72 | self.transport = transport 73 | self.site_url = site_url 74 | self._properties = {"client": {"xlogin": {}}} 75 | if login_successful: 76 | self.add_login_info({"test_login": "test_logged_in"}) 77 | 78 | @property 79 | def properties(self) -> Dict: 80 | return {**self.__dict__, **self._properties} 81 | 82 | def add_login_info(self, login: Dict) -> None: 83 | self._properties["client"]["xlogin"] = login 84 | 85 | def new_job(self, job_description: Dict, inputs: List) -> FakeJob: 86 | return FakeJob(transport=self.transport, job_url="test_job_url") 87 | -------------------------------------------------------------------------------- /tests/unit/cli/preferences: -------------------------------------------------------------------------------- 1 | # User preferences for UCC 2 | 3 | authentication-method=USERNAME 4 | username=demouser 5 | password=test123 6 | 7 | verbose=true 8 | 9 | # 10 | # The address(es) of the registries to contact 11 | # (space separated list) 12 | registry=https://localhost:8080/DEMO-SITE/rest/registries/default_registry 13 | contact-registry=true 14 | 15 | # 16 | # default directory for output 17 | # 18 | output=/tmp 19 | -------------------------------------------------------------------------------- /tests/unit/cli/test_base.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.cli.base as base 4 | from pyunicore.credentials import UsernamePassword 5 | 6 | 7 | class TestBase(unittest.TestCase): 8 | def test_base_setup(self): 9 | cmd = base.Base() 10 | cmd.config_file = "tests/unit/cli/preferences" 11 | cmd.load_user_properties() 12 | self.assertEqual( 13 | "https://localhost:8080/DEMO-SITE/rest/registries/default_registry", 14 | cmd.config["registry"], 15 | ) 16 | cmd.create_credential() 17 | self.assertTrue(isinstance(cmd.credential, UsernamePassword)) 18 | 19 | 20 | if __name__ == "__main__": 21 | unittest.main() 22 | -------------------------------------------------------------------------------- /tests/unit/cli/test_io.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.cli.io as io 4 | 5 | 6 | class TestIO(unittest.TestCase): 7 | 8 | def test_split_path(self): 9 | tests = { 10 | "/foo/*": ("/foo", "*"), 11 | "/foo/test.txt": ("/foo", "test.txt"), 12 | "test.txt": ("/", "test.txt"), 13 | "/test.txt": ("/", "test.txt"), 14 | "/foo/bar/test.txt": ("/foo/bar", "test.txt"), 15 | } 16 | for p in tests: 17 | base, pattern = io.split_path(p) 18 | self.assertEqual((base, pattern), tests[p]) 19 | 20 | def test_normalize(self): 21 | tests = { 22 | "/foo//bar": "/foo/bar", 23 | } 24 | for p in tests: 25 | self.assertEqual(io.normalized(p), tests[p]) 26 | 27 | 28 | if __name__ == "__main__": 29 | unittest.main() 30 | -------------------------------------------------------------------------------- /tests/unit/cli/test_main.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.cli.main as main 4 | 5 | 6 | class TestMain(unittest.TestCase): 7 | def test_help(self): 8 | main.help() 9 | main.show_version() 10 | for cmd in main._commands: 11 | print("\n*** %s *** " % cmd) 12 | c = main.get_command(cmd) 13 | print(c.get_synopsis()) 14 | c.parser.print_usage() 15 | c.parser.print_help() 16 | 17 | def test_run_args(self): 18 | main.run([]) 19 | main.run(["--version"]) 20 | main.run(["--help"]) 21 | try: 22 | main.run(["no-such-cmd"]) 23 | self.fail() 24 | except ValueError: 25 | pass 26 | 27 | 28 | if __name__ == "__main__": 29 | unittest.main() 30 | -------------------------------------------------------------------------------- /tests/unit/helpers/connection/test_registry_helper.py: -------------------------------------------------------------------------------- 1 | import functools 2 | from typing import Dict 3 | 4 | import pytest 5 | 6 | import pyunicore.client 7 | import pyunicore.credentials as credentials 8 | import pyunicore.helpers.connection.registry as _registry 9 | import tests.testing as testing 10 | 11 | 12 | @pytest.fixture() 13 | def credential(): 14 | return credentials.UsernamePassword(username="test_user", password="test_password") 15 | 16 | 17 | def create_fake_registry(contains: Dict[str, str]) -> functools.partial: 18 | return functools.partial( 19 | testing.FakeRegistry, 20 | contains=contains, 21 | ) 22 | 23 | 24 | def create_fake_client(login_successful: bool) -> functools.partial: 25 | return functools.partial( 26 | testing.FakeClient, 27 | login_successful=login_successful, 28 | ) 29 | 30 | 31 | def test_connect_to_registry(monkeypatch): 32 | monkeypatch.setattr(pyunicore.client, "Transport", testing.FakeTransport) 33 | 34 | registry_url = "test_registry_url" 35 | creds = credentials.UsernamePassword( 36 | username="test_user", 37 | password="test_password", 38 | ) 39 | 40 | result = _registry.connect_to_registry( 41 | registry_url=registry_url, 42 | credential=creds, 43 | ) 44 | 45 | assert isinstance(result, pyunicore.client.Registry) 46 | 47 | 48 | @pytest.mark.parametrize( 49 | ("login_successful", "expected"), 50 | [ 51 | (False, credentials.AuthenticationFailedException()), 52 | (True, testing.FakeClient), 53 | ], 54 | ) 55 | def test_connect_to_site_from_registry(monkeypatch, login_successful, expected): 56 | monkeypatch.setattr(pyunicore.client, "Transport", testing.FakeTransport) 57 | monkeypatch.setattr( 58 | pyunicore.client, 59 | "Registry", 60 | create_fake_registry(contains={"test_site": "test_api_url"}), 61 | ) 62 | monkeypatch.setattr( 63 | pyunicore.client, 64 | "Client", 65 | create_fake_client(login_successful=login_successful), 66 | ) 67 | 68 | registry_url = "test_registry_url" 69 | site = "test_site" 70 | creds = credentials.UsernamePassword( 71 | username="test_user", 72 | password="test_password", 73 | ) 74 | 75 | with testing.expect_raise_if_exception(expected): 76 | result = _registry.connect_to_site_from_registry( 77 | registry_url=registry_url, 78 | site_name=site, 79 | credential=creds, 80 | ) 81 | 82 | assert isinstance(result, expected) 83 | 84 | 85 | @pytest.mark.parametrize( 86 | ("site", "expected"), 87 | [ 88 | ("test_site", "test_api_url"), 89 | ("test_unavailable_site", ValueError()), 90 | ], 91 | ) 92 | def test_get_site_api_url_from_registry(monkeypatch, credential, site, expected): 93 | monkeypatch.setattr( 94 | pyunicore.client, 95 | "Registry", 96 | create_fake_registry(contains={"test_site": "test_api_url"}), 97 | ) 98 | 99 | with testing.expect_raise_if_exception(expected): 100 | result = _registry._get_site_api_url( 101 | site=site, 102 | credential=credential, 103 | registry_url="test_registry_url", 104 | ) 105 | 106 | assert result == expected 107 | -------------------------------------------------------------------------------- /tests/unit/helpers/connection/test_site.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | import pytest 4 | 5 | import pyunicore.client as pyunicore 6 | import pyunicore.credentials as credentials 7 | import pyunicore.helpers.connection.site as _connect 8 | import tests.testing as testing 9 | 10 | 11 | @pytest.fixture() 12 | def transport(): 13 | return testing.FakeTransport() 14 | 15 | 16 | def create_fake_client(login_successful: bool) -> functools.partial: 17 | return functools.partial( 18 | testing.FakeClient, 19 | login_successful=login_successful, 20 | ) 21 | 22 | 23 | @pytest.mark.parametrize( 24 | ("login_successful", "expected"), 25 | [ 26 | (False, credentials.AuthenticationFailedException()), 27 | (True, testing.FakeClient), 28 | ], 29 | ) 30 | def test_connect_to_site(monkeypatch, login_successful, expected): 31 | monkeypatch.setattr(pyunicore, "Transport", testing.FakeTransport) 32 | monkeypatch.setattr( 33 | pyunicore, 34 | "Client", 35 | create_fake_client(login_successful=login_successful), 36 | ) 37 | 38 | api_url = "test-api-url" 39 | creds = credentials.UsernamePassword( 40 | username="test_user", 41 | password="test_password", 42 | ) 43 | 44 | with testing.expect_raise_if_exception(expected): 45 | result = _connect.connect_to_site( 46 | site_api_url=api_url, 47 | credential=creds, 48 | ) 49 | 50 | assert isinstance(result, expected) 51 | 52 | 53 | @pytest.mark.parametrize( 54 | ("login", "expected"), 55 | [ 56 | ({}, True), 57 | ({"test_login_info": "test_login"}, False), 58 | ], 59 | ) 60 | def test_authentication_failed(login, expected): 61 | client = testing.FakeClient() 62 | client.add_login_info(login) 63 | 64 | result = _connect._authentication_failed(client) 65 | 66 | assert result == expected 67 | -------------------------------------------------------------------------------- /tests/unit/helpers/jobs/test_data.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import pyunicore.helpers.jobs.data as data 4 | 5 | 6 | class TestCredentials: 7 | def test_to_dict(self): 8 | credentials = data.Credentials("user", "password") 9 | expected = { 10 | "Username": "user", 11 | "Password": "password", 12 | } 13 | 14 | result = credentials.to_dict() 15 | 16 | assert result == expected 17 | 18 | 19 | class TestImport: 20 | @pytest.mark.parametrize( 21 | ("import_", "expected"), 22 | [ 23 | ( 24 | data.Import( 25 | from_="here", 26 | to="there", 27 | ), 28 | {"From": "here", "To": "there", "FailOnError": "true"}, 29 | ), 30 | ( 31 | data.Import( 32 | from_="here", 33 | to="there", 34 | fail_on_error=False, 35 | ), 36 | {"From": "here", "To": "there", "FailOnError": "false"}, 37 | ), 38 | ( 39 | data.Import( 40 | from_="here", 41 | to="there", 42 | data="test", 43 | ), 44 | { 45 | "From": "here", 46 | "To": "there", 47 | "FailOnError": "true", 48 | "Data": "test", 49 | }, 50 | ), 51 | ( 52 | data.Import( 53 | from_="here", 54 | to="there", 55 | credentials=data.Credentials("user", "password"), 56 | ), 57 | { 58 | "From": "here", 59 | "To": "there", 60 | "FailOnError": "true", 61 | "Credentials": {"Username": "user", "Password": "password"}, 62 | }, 63 | ), 64 | ], 65 | ) 66 | def test_to_dict(self, import_, expected): 67 | result = import_.to_dict() 68 | 69 | assert result == expected 70 | 71 | 72 | class TestExport: 73 | def test_to_dict(self): 74 | export_ = data.Export( 75 | from_="there", 76 | to="here", 77 | ) 78 | expected = {"From": "there", "To": "here"} 79 | 80 | result = export_.to_dict() 81 | 82 | assert result == expected 83 | -------------------------------------------------------------------------------- /tests/unit/helpers/jobs/test_job_description.py: -------------------------------------------------------------------------------- 1 | import pyunicore.helpers.jobs.description as description 2 | import pyunicore.helpers.jobs.resources as resources 3 | 4 | 5 | class TestJobDescription: 6 | def test_to_dict(self): 7 | res = resources.Resources(nodes=2, queue="batch") 8 | job = description.Description( 9 | executable="test-executable", 10 | project="test-project", 11 | resources=res, 12 | ) 13 | expected = { 14 | "Executable": "test-executable", 15 | "IgnoreNonZeroExitCode": "false", 16 | "Job type": "normal", 17 | "Project": "test-project", 18 | "Resources": {"Nodes": 2, "Queue": "batch"}, 19 | "RunUserPostcommandOnLoginNode": "true", 20 | "RunUserPrecommandOnLoginNode": "true", 21 | "Stderr": "stderr", 22 | "Stdout": "stdout", 23 | "UserPostcommandIgnoreNonZeroExitcode": "false", 24 | "UserPrecommandIgnoreNonZeroExitcode": "false", 25 | "haveClientStageIn": "false", 26 | } 27 | 28 | result = job.to_dict() 29 | 30 | assert result == expected 31 | -------------------------------------------------------------------------------- /tests/unit/helpers/test_api_object.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Dict 3 | 4 | import pytest 5 | 6 | import pyunicore.helpers._api_object as api_object 7 | 8 | 9 | @dataclasses.dataclass 10 | class ApiObject(api_object.ApiRequestObject): 11 | test: str = "test" 12 | 13 | def _to_dict(self) -> Dict: 14 | return dataclasses.asdict(self) 15 | 16 | 17 | @pytest.mark.parametrize( 18 | ("kwargs", "expected"), 19 | [ 20 | ( 21 | {"any": "value"}, 22 | {"any": "value"}, 23 | ), 24 | ( 25 | {"any": None}, 26 | {}, 27 | ), 28 | ( 29 | {"any": ApiObject()}, 30 | {"any": {"test": "test"}}, 31 | ), 32 | # bool needs to be converted to lower-case string 33 | ( 34 | {"any": True}, 35 | {"any": "true"}, 36 | ), 37 | ( 38 | {"any": False}, 39 | {"any": "false"}, 40 | ), 41 | ], 42 | ) 43 | def test_create_dict_with_not_none_values(kwargs, expected): 44 | result = api_object._create_dict_with_not_none_values(kwargs) 45 | 46 | assert result == expected 47 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/activities/loops/conftest.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import pytest 4 | 5 | from pyunicore.helpers.workflows import activities as _activities 6 | from pyunicore.helpers.workflows import transition 7 | from pyunicore.helpers.workflows.activities import loops 8 | 9 | 10 | @pytest.fixture(scope="session") 11 | def loop_body() -> loops.Body: 12 | activities = [ 13 | _activities.ModifyVariable( 14 | id="test-modify-variable-id", 15 | variable_name="test-variable", 16 | expression="test-expression", 17 | ) 18 | ] 19 | transitions = [transition.Transition(from_="here", to="there", condition="test-condition")] 20 | return loops.Body( 21 | activities=activities, 22 | transitions=transitions, 23 | condition="test-body-condition", 24 | ) 25 | 26 | 27 | @pytest.fixture(scope="session") 28 | def expected_loop_body() -> Dict: 29 | return { 30 | "activities": [ 31 | { 32 | "id": "test-modify-variable-id", 33 | "type": "ModifyVariable", 34 | "variableName": "test-variable", 35 | "expression": "test-expression", 36 | }, 37 | ], 38 | "transitions": [ 39 | {"from": "here", "to": "there", "condition": "test-condition"}, 40 | ], 41 | "condition": "test-body-condition", 42 | } 43 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/activities/loops/test_for_loop.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pyunicore.helpers.workflows import variable 4 | from pyunicore.helpers.workflows.activities import loops 5 | 6 | 7 | class TestChunking: 8 | def test_chunksize_and_chunking_formula_given(self): 9 | with pytest.raises(ValueError): 10 | loops.Chunking( 11 | chunksize=3, 12 | type=loops.ChunkingType.Normal, 13 | chunksize_formula="if(TOTAL_SIZE>50*1024)return 5*1024 else return 2048;", # noqa 14 | ) 15 | 16 | 17 | class TestFor: 18 | @pytest.mark.parametrize( 19 | ("range", "chunking", "expected_range", "expected_chunking"), 20 | [ 21 | # Test case: values used for iteration. 22 | (loops.Values([1, 2, 3]), None, {"values": [1, 2, 3]}, None), 23 | # Test case: Normal chunking with size 3. 24 | ( 25 | loops.Values([1, 2, 3]), 26 | loops.Chunking( 27 | chunksize=3, 28 | type=loops.ChunkingType.Normal, 29 | ), 30 | {"values": [1, 2, 3]}, 31 | { 32 | "chunksize": 3, 33 | "type": "NORMAL", 34 | }, 35 | ), 36 | # Test case: Size chunking with size 3 kbytes. 37 | ( 38 | loops.Values([1, 2, 3]), 39 | loops.Chunking( 40 | chunksize=3, 41 | type=loops.ChunkingType.Size, 42 | ), 43 | {"values": [1, 2, 3]}, 44 | { 45 | "chunksize": 3, 46 | "type": "SIZE", 47 | }, 48 | ), 49 | # Test case: Normal chunking with size 3 and filename format. 50 | ( 51 | loops.Values([1, 2, 3]), 52 | loops.Chunking( 53 | chunksize=3, 54 | type=loops.ChunkingType.Normal, 55 | filename_format="file_{0}.pdf", 56 | ), 57 | {"values": [1, 2, 3]}, 58 | { 59 | "chunksize": 3, 60 | "type": "NORMAL", 61 | "filename_format": "file_{0}.pdf", 62 | }, 63 | ), 64 | # Test case: Size with chunksize formula. 65 | ( 66 | loops.Values([1, 2, 3]), 67 | loops.Chunking( 68 | type=loops.ChunkingType.Size, 69 | chunksize_formula="if(TOTAL_SIZE>50*1024)return 5*1024 else return 2048;", # noqa 70 | ), 71 | {"values": [1, 2, 3]}, 72 | { 73 | "type": "SIZE", 74 | "chunksize_formula": "if(TOTAL_SIZE>50*1024)return 5*1024 else return 2048;", # noqa 75 | }, 76 | ), 77 | # Test case: Variables used for iteration. 78 | ( 79 | loops.Variables( 80 | [ 81 | loops.Variable( 82 | name="x", 83 | type=variable.VariableType.Integer, 84 | initial_value=1, 85 | expression="x++", 86 | end_condition="x<2", 87 | ) 88 | ] 89 | ), 90 | None, 91 | { 92 | "variables": [ 93 | { 94 | "variable_name": "x", 95 | "type": "INTEGER", 96 | "start_value": 1, 97 | "expression": "x++", 98 | "end_condition": "x<2", 99 | } 100 | ] 101 | }, 102 | None, 103 | ), 104 | # Test case: Files used for iteration. 105 | ( 106 | loops.Files( 107 | [ 108 | loops.File( 109 | base="https://mysite/rest/core/storages/my_storage/files/pdf/", # noqa 110 | include=["*.pdf"], 111 | exclude=[ 112 | "unused1.pdf", 113 | "unused2.pdf", 114 | ], 115 | recurse=True, 116 | indirection=True, 117 | ) 118 | ] 119 | ), 120 | None, 121 | { 122 | "file_sets": [ 123 | { 124 | "base": "https://mysite/rest/core/storages/my_storage/files/pdf/", # noqa 125 | "include": ["*.pdf"], 126 | "exclude": [ 127 | "unused1.pdf", 128 | "unused2.pdf", 129 | ], 130 | "recurse": "true", 131 | "indirection": "true", 132 | } 133 | ] 134 | }, 135 | None, 136 | ), 137 | ], 138 | ) 139 | def test_to_dict( 140 | self, 141 | loop_body, 142 | expected_loop_body, 143 | range, 144 | chunking, 145 | expected_range, 146 | expected_chunking, 147 | ): 148 | loop = loops.ForEach( 149 | id="test-for-each-loop-id", 150 | body=loop_body, 151 | iterator_name="test-name", 152 | range=range, 153 | chunking=chunking, 154 | ) 155 | 156 | expected = { 157 | "id": "test-for-each-loop-id", 158 | "type": "FOR_EACH", 159 | "iterator_name": "test-name", 160 | **expected_range, 161 | "body": expected_loop_body, 162 | } 163 | 164 | if chunking is not None: 165 | expected["chunking"] = expected_chunking 166 | 167 | result = loop.to_dict() 168 | 169 | assert result == expected 170 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/activities/loops/test_repeat_until_loop.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows import variable 2 | from pyunicore.helpers.workflows.activities import loops 3 | 4 | 5 | class TestRepeatUntil: 6 | def test_to_dict(self, loop_body, expected_loop_body): 7 | variables = [ 8 | variable.Variable( 9 | name="test-variable", 10 | type=variable.VariableType.Integer, 11 | initial_value=1, 12 | ) 13 | ] 14 | loop = loops.RepeatUntil( 15 | id="test-repeat-until-loop-id", 16 | variables=variables, 17 | body=loop_body, 18 | ) 19 | 20 | expected = { 21 | "id": "test-repeat-until-loop-id", 22 | "type": "REPEAT_UNTIL", 23 | "variables": [{"name": "test-variable", "type": "INTEGER", "initial_value": 1}], 24 | "body": expected_loop_body, 25 | } 26 | 27 | result = loop.to_dict() 28 | 29 | assert result == expected 30 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/activities/loops/test_while_loop.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows import variable 2 | from pyunicore.helpers.workflows.activities import loops 3 | 4 | 5 | class TestWhileLoop: 6 | def test_to_dict(self, loop_body, expected_loop_body): 7 | variables = [ 8 | variable.Variable( 9 | name="test-variable", 10 | type=variable.VariableType.Integer, 11 | initial_value=1, 12 | ) 13 | ] 14 | loop = loops.While( 15 | id="test-while-loop-id", 16 | variables=variables, 17 | body=loop_body, 18 | ) 19 | 20 | expected = { 21 | "id": "test-while-loop-id", 22 | "type": "WHILE", 23 | "variables": [{"name": "test-variable", "type": "INTEGER", "initial_value": 1}], 24 | "body": expected_loop_body, 25 | } 26 | 27 | result = loop.to_dict() 28 | 29 | assert result == expected 30 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/activities/test_activity.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows.activities import activity 2 | 3 | 4 | class TestStart: 5 | def test_to_dict(self): 6 | start = activity.Start( 7 | id="test-id", 8 | ) 9 | expected = { 10 | "id": "test-id", 11 | "type": "START", 12 | } 13 | 14 | result = start.to_dict() 15 | 16 | assert result == expected 17 | 18 | 19 | class TestSplit: 20 | def test_to_dict(self): 21 | split = activity.Split( 22 | id="test-id", 23 | ) 24 | expected = { 25 | "id": "test-id", 26 | "type": "Split", 27 | } 28 | 29 | result = split.to_dict() 30 | 31 | assert result == expected 32 | 33 | 34 | class TestBranch: 35 | def test_to_dict(self): 36 | branch = activity.Branch( 37 | id="test-id", 38 | ) 39 | expected = { 40 | "id": "test-id", 41 | "type": "BRANCH", 42 | } 43 | 44 | result = branch.to_dict() 45 | 46 | assert result == expected 47 | 48 | 49 | class TestMerge: 50 | def test_to_dict(self): 51 | merge = activity.Merge( 52 | id="test-id", 53 | ) 54 | expected = { 55 | "id": "test-id", 56 | "type": "Merge", 57 | } 58 | 59 | result = merge.to_dict() 60 | 61 | assert result == expected 62 | 63 | 64 | class TestSynchronize: 65 | def test_to_dict(self): 66 | synchronize = activity.Synchronize( 67 | id="test-id", 68 | ) 69 | expected = { 70 | "id": "test-id", 71 | "type": "Synchronize", 72 | } 73 | 74 | result = synchronize.to_dict() 75 | 76 | assert result == expected 77 | 78 | 79 | class TestHold: 80 | def test_to_dict(self): 81 | hold = activity.Hold( 82 | id="test-id", 83 | ) 84 | expected = { 85 | "id": "test-id", 86 | "type": "HOLD", 87 | } 88 | 89 | result = hold.to_dict() 90 | 91 | assert result == expected 92 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/activities/test_job_activity.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers import jobs 2 | from pyunicore.helpers.workflows.activities import job 3 | 4 | 5 | class TestJob: 6 | def test_to_dict(self): 7 | user_preferences = job.UserPreferences( 8 | role="test-role", 9 | uid="test-uid", 10 | group="test-group", 11 | supplementary_groups="test-groups", 12 | ) 13 | options = [job.Option.IgnoreFailure(True), job.Option.MaxResubmits(2)] 14 | description = jobs.Description( 15 | executable="test-executable", 16 | project="test-project", 17 | ) 18 | job_ = job.Job( 19 | id="test-id", 20 | description=description, 21 | site_name="test-site", 22 | user_preferences=user_preferences, 23 | options=options, 24 | ) 25 | expected = { 26 | "id": "test-id", 27 | "type": "JOB", 28 | "job": { 29 | "Executable": "test-executable", 30 | "IgnoreNonZeroExitCode": "false", 31 | "Job type": "normal", 32 | "Project": "test-project", 33 | "Resources": {}, 34 | "RunUserPostcommandOnLoginNode": "true", 35 | "RunUserPrecommandOnLoginNode": "true", 36 | "Site name": "test-site", 37 | "Stderr": "stderr", 38 | "Stdout": "stdout", 39 | "UserPostcommandIgnoreNonZeroExitcode": "false", 40 | "UserPrecommandIgnoreNonZeroExitcode": "false", 41 | "haveClientStageIn": "false", 42 | "User preferences": { 43 | "role": "test-role", 44 | "uid": "test-uid", 45 | "group": "test-group", 46 | "supplementaryGroups": "test-groups", 47 | }, 48 | }, 49 | "options": {"IGNORE_FAILURE": "true", "MAX_RESUBMITS": 2}, 50 | } 51 | 52 | result = job_.to_dict() 53 | 54 | assert result == expected 55 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/activities/test_modify_variable.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows.activities import modify_variable 2 | 3 | 4 | class TestModifyVariable: 5 | def test_to_dict(self): 6 | variable = modify_variable.ModifyVariable( 7 | id="test-id", variable_name="x", expression="x + 1" 8 | ) 9 | expected = { 10 | "id": "test-id", 11 | "type": "ModifyVariable", 12 | "variableName": "x", 13 | "expression": "x + 1", 14 | } 15 | 16 | result = variable.to_dict() 17 | 18 | assert result == expected 19 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/test_transition.py: -------------------------------------------------------------------------------- 1 | from pyunicore.helpers.workflows import transition 2 | 3 | 4 | class TestTransition: 5 | def test_to_dict(self): 6 | transition_ = transition.Transition( 7 | from_="here", 8 | to="there", 9 | condition="test-condition", 10 | ) 11 | expected = { 12 | "from": "here", 13 | "to": "there", 14 | "condition": "test-condition", 15 | } 16 | 17 | result = transition_.to_dict() 18 | 19 | assert result == expected 20 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/test_variable.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pyunicore.helpers.workflows import variable 4 | from tests import testing 5 | 6 | 7 | class TestVariable: 8 | @pytest.mark.parametrize( 9 | ("type", "initial_value", "expected"), 10 | [ 11 | ( 12 | variable.VariableType.Integer, 13 | 1, 14 | { 15 | "name": "test-variable", 16 | "type": "INTEGER", 17 | "initial_value": 1, 18 | }, 19 | ), 20 | ( 21 | variable.VariableType.String, 22 | "test", 23 | { 24 | "name": "test-variable", 25 | "type": "STRING", 26 | "initial_value": "test", 27 | }, 28 | ), 29 | ( 30 | variable.VariableType.Float, 31 | 1.0, 32 | { 33 | "name": "test-variable", 34 | "type": "FLOAT", 35 | "initial_value": 1.0, 36 | }, 37 | ), 38 | ( 39 | variable.VariableType.Boolean, 40 | True, 41 | { 42 | "name": "test-variable", 43 | "type": "BOOLEAN", 44 | "initial_value": "true", 45 | }, 46 | ), 47 | ( 48 | variable.VariableType.Boolean, 49 | False, 50 | { 51 | "name": "test-variable", 52 | "type": "BOOLEAN", 53 | "initial_value": "false", 54 | }, 55 | ), 56 | # Test case: Given type not supported. 57 | ( 58 | dict, 59 | {}, 60 | ValueError(), 61 | ), 62 | # Test case: Initial value not of correct type. 63 | ( 64 | variable.VariableType.Float, 65 | "wrong type", 66 | ValueError(), 67 | ), 68 | ], 69 | ) 70 | def test_to_dict(self, type, initial_value, expected): 71 | with testing.expect_raise_if_exception(expected): 72 | var = variable.Variable( 73 | name="test-variable", 74 | type=type, 75 | initial_value=initial_value, 76 | ) 77 | 78 | assert var.to_dict() == expected 79 | -------------------------------------------------------------------------------- /tests/unit/helpers/workflows/test_workflow_description.py: -------------------------------------------------------------------------------- 1 | import pyunicore.helpers.workflows as workflows 2 | 3 | 4 | class TestWorkflowDescription: 5 | def test_to_dict(self): 6 | activities = [workflows.activities.Start(id="test-start")] 7 | transitions = [ 8 | workflows.Transition( 9 | from_="here", 10 | to="there", 11 | ) 12 | ] 13 | variables = [ 14 | workflows.Variable( 15 | name="test-variable", 16 | type=workflows.VariableType.Integer, 17 | initial_value=1, 18 | ) 19 | ] 20 | 21 | workflow = workflows.Description( 22 | activities=activities, 23 | transitions=transitions, 24 | variables=variables, 25 | ) 26 | expected = { 27 | "activities": [{"id": "test-start", "type": "START"}], 28 | "transitions": [{"from": "here", "to": "there"}], 29 | "variables": [{"initial_value": 1, "name": "test-variable", "type": "INTEGER"}], 30 | } 31 | 32 | result = workflow.to_dict() 33 | 34 | assert result == expected 35 | -------------------------------------------------------------------------------- /tests/unit/test_cwl1.py: -------------------------------------------------------------------------------- 1 | import json 2 | import unittest 3 | 4 | import pyunicore.cwl.cwlconverter as cwlconverter 5 | import pyunicore.cwl.cwltool as cwltool 6 | 7 | 8 | class TestCWL1(unittest.TestCase): 9 | def setUp(self): 10 | pass 11 | 12 | def test_convert_echo(self): 13 | print("*** test_convert_echo") 14 | cwl_doc, cwl_input_object = cwltool.read_cwl_files( 15 | "tests/cwldocs/echo.cwl", "tests/cwldocs/echo.params" 16 | ) 17 | u_job, files, outputs = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_input_object) 18 | print(json.dumps(u_job, indent=2)) 19 | self.assertEqual("echo", u_job["Executable"]) 20 | self.assertEqual('"hello world!"', u_job["Arguments"][0]) 21 | self.assertEqual("two", u_job["Arguments"][1]) 22 | self.assertEqual("42", u_job["Arguments"][2]) 23 | self.assertEqual("1,2,3", u_job["Arguments"][3]) 24 | self.assertEqual(["-x", "7", "-x", "8"], u_job["Arguments"][4:9]) 25 | self.assertEqual("my_out", u_job["Stdout"]) 26 | self.assertEqual("my_err", u_job["Stderr"]) 27 | self.assertEqual(0, len(files)) 28 | 29 | def test_convert_fileparam(self): 30 | print("*** test_convert_fileparam") 31 | cwl_doc, cwl_input_object = cwltool.read_cwl_files( 32 | "tests/cwldocs/fileinput.cwl", "tests/cwldocs/fileinput.params" 33 | ) 34 | u_job, files, outputs = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_input_object) 35 | print(json.dumps(u_job, indent=2)) 36 | self.assertEqual("--file1=test.sh", u_job["Arguments"][0]) 37 | self.assertEqual(["--file2", "file2"], u_job["Arguments"][1:3]) 38 | self.assertEqual(2, len(files)) 39 | self.assertTrue("test.sh" in files) 40 | self.assertTrue("file2" in files) 41 | 42 | def test_convert_fileparam_with_remotes(self): 43 | print("*** test_convert_fileparam_with_remotes") 44 | cwl_doc, cwl_input_object = cwltool.read_cwl_files( 45 | "tests/cwldocs/fileinput_remote.cwl", 46 | "tests/cwldocs/fileinput_remote.params", 47 | ) 48 | u_job, files, outputs = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_input_object) 49 | print(json.dumps(u_job, indent=2)) 50 | self.assertEqual("--file1=test.sh", u_job["Arguments"][0]) 51 | self.assertEqual(["--file2", "file2"], u_job["Arguments"][1:3]) 52 | self.assertEqual("some_remote_file", u_job["Arguments"][3]) 53 | self.assertEqual("file.txt", u_job["Arguments"][4]) 54 | self.assertEqual(2, len(files)) 55 | self.assertTrue("test.sh" in files) 56 | self.assertTrue("file2" in files) 57 | remotes = u_job["Imports"] 58 | self.assertEqual(2, len(remotes)) 59 | print(json.dumps(u_job, indent=2)) 60 | 61 | def test_handle_directory_param(self): 62 | print("*** test_handle_directory_param") 63 | cwl_doc, cwl_input_object = cwltool.read_cwl_files( 64 | "tests/cwldocs/directoryinput.cwl", 65 | "tests/cwldocs/directoryinput.params", 66 | ) 67 | u_job, files, outputs = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_input_object) 68 | self.assertEqual("--input=tests/cwldocs", u_job["Arguments"][0]) 69 | self.assertEqual(1, len(files)) 70 | 71 | def test_convert_array_inputs(self): 72 | print("*** test_convert_array_inputs") 73 | cwl_doc, cwl_input_object = cwltool.read_cwl_files( 74 | "tests/cwldocs/array-inputs.cwl", 75 | "tests/cwldocs/array-inputs.params", 76 | ) 77 | u_job, files, outputs = cwlconverter.convert_cmdline_tool(cwl_doc, cwl_input_object) 78 | print(json.dumps(u_job, indent=2)) 79 | args = u_job["Arguments"] 80 | self.assertEqual(8, len(args)) 81 | self.assertEqual(["-A", "one", "two", "three"], args[0:4]) 82 | self.assertEqual(["-B=four", "-B=five", "-B=six"], args[4:7]) 83 | self.assertEqual("-C=seven,eight,nine", args[7]) 84 | 85 | def test_resolve_input_files(self): 86 | print("*** test_resolve_input_files") 87 | pass 88 | 89 | 90 | if __name__ == "__main__": 91 | unittest.main() 92 | -------------------------------------------------------------------------------- /tests/unit/test_jwt_tokens.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pyunicore.credentials as uc_credentials 4 | 5 | 6 | class TestJWTCredentials(unittest.TestCase): 7 | def setUp(self): 8 | pass 9 | 10 | def test_hs256(self): 11 | print("*** test_hs256") 12 | credential = uc_credentials.JWTToken( 13 | "CN=Demouser", 14 | "CN=My Service", 15 | secret="test123", 16 | algorithm="HS256", 17 | etd=True, 18 | ) 19 | print(credential.get_token()) 20 | 21 | 22 | if __name__ == "__main__": 23 | unittest.main() 24 | -------------------------------------------------------------------------------- /tests/unit/test_uftpfs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from os import environ 3 | 4 | from pyunicore.uftp.uftpfs import UFTPOpener 5 | 6 | 7 | class TestUFTPFS(unittest.TestCase): 8 | def setUp(self): 9 | pass 10 | 11 | def test_parse_url(self): 12 | print("*** test_parse_url") 13 | u1 = "localhost:9000/rest/auth/TEST:/data-dir" 14 | auth_url, base_dir = UFTPOpener()._parse(u1) 15 | self.assertEqual("https://localhost:9000/rest/auth/TEST", auth_url) 16 | self.assertEqual("/data-dir", base_dir) 17 | 18 | def test_credential_username_password(self): 19 | print("*** test_credential_username_password") 20 | parse_result = P() 21 | cred = UFTPOpener()._create_credential(parse_result) 22 | self.assertEqual(cred.username, "demouser") 23 | 24 | def test_credential_token(self): 25 | print("*** test_credential_token") 26 | parse_result = P() 27 | parse_result.username = None 28 | parse_result.password = None 29 | parse_result.params["token"] = "some_token" 30 | cred = UFTPOpener()._create_credential(parse_result) 31 | self.assertEqual(cred.token, "some_token") 32 | parse_result.params["token"] = "@tests/unit/token.txt" 33 | cred = UFTPOpener()._create_credential(parse_result) 34 | self.assertEqual(cred.token, "some_token") 35 | parse_result.params["token"] = "@@MY_VAR" 36 | environ["MY_VAR"] = "some_token" 37 | cred = UFTPOpener()._create_credential(parse_result) 38 | self.assertEqual(cred.token, "some_token") 39 | 40 | 41 | class P: 42 | def __init__(self): 43 | self.username = "demouser" 44 | self.password = "test123" 45 | self.params = {} 46 | 47 | 48 | if __name__ == "__main__": 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from base64 import b64encode 3 | 4 | import pyunicore.credentials as uc_credentials 5 | from pyunicore.client import Transport 6 | 7 | 8 | class TestCredentials(unittest.TestCase): 9 | def setUp(self): 10 | pass 11 | 12 | def test_username_password(self): 13 | print("*** test_username_password") 14 | credential = uc_credentials.UsernamePassword("demouser", "test123") 15 | self.assertEqual( 16 | "Basic " + b64encode(b"demouser:test123").decode("ascii"), 17 | credential.get_auth_header(), 18 | ) 19 | 20 | def test_username_password_via_factory(self): 21 | print("*** test_username_password_via_factory") 22 | credential = uc_credentials.create_credential("demouser", "test123") 23 | self.assertEqual( 24 | "Basic " + b64encode(b"demouser:test123").decode("ascii"), 25 | credential.get_auth_header(), 26 | ) 27 | 28 | def test_oidc_token(self): 29 | print("*** test_oidc_token") 30 | credential = uc_credentials.OIDCToken("test123") 31 | self.assertEqual("Bearer test123", credential.get_auth_header()) 32 | 33 | def test_oidc_token_via_factory(self): 34 | print("*** test_oidc_token_via_factory") 35 | credential = uc_credentials.create_credential(token="test123") 36 | self.assertEqual("Bearer test123", credential.get_auth_header()) 37 | 38 | def test_oidc_token_with_refresh(self): 39 | print("*** test_oidc_token_with_refresh") 40 | refresh_handler = MockRefresh() 41 | credential = uc_credentials.OIDCToken("test123", refresh_handler) 42 | self.assertEqual("Bearer foobar", credential.get_auth_header()) 43 | 44 | def test_basic_token(self): 45 | print("*** test_basic_token") 46 | credential = uc_credentials.BasicToken("test123") 47 | self.assertEqual("Basic test123", credential.get_auth_header()) 48 | 49 | def test_transport(self): 50 | print("*** test_transport") 51 | token_str = b64encode(b"demouser:test123").decode("ascii") 52 | header_val = "Basic " + token_str 53 | credential = uc_credentials.UsernamePassword("demouser", "test123") 54 | transport = Transport(credential) 55 | self.assertEqual(header_val, transport._headers({})["Authorization"]) 56 | 57 | 58 | class MockRefresh(uc_credentials.RefreshHandler): 59 | def refresh_token(self): 60 | return "foobar" 61 | 62 | 63 | if __name__ == "__main__": 64 | unittest.main() 65 | -------------------------------------------------------------------------------- /tests/unit/token.txt: -------------------------------------------------------------------------------- 1 | 2 | some_token 3 | --------------------------------------------------------------------------------