├── .github └── workflows │ ├── python-publish.yml │ └── testing.yml ├── .gitignore ├── .mypy.ini ├── .readthedocs.yml ├── LICENSE ├── README.md ├── async-example.md ├── docs ├── Makefile └── source │ ├── authorization.rst │ ├── conf.py │ ├── index.rst │ ├── reference_async.rst │ ├── reference_async_v2.rst │ ├── reference_auth.rst │ ├── reference_basic.rst │ ├── reference_cli.rst │ ├── reference_sync_v2.rst │ ├── reference_v1_index.rst │ ├── reference_v2_index.rst │ ├── tutorial_async.rst │ ├── tutorial_basic_v1.rst │ ├── tutorial_basic_v2.rst │ ├── tutorial_cli.rst │ ├── tutorial_errors.rst │ ├── tutorial_index.rst │ └── tutorial_logging.rst ├── firecrest ├── Authorization.py ├── FirecrestException.py ├── __init__.py ├── cli │ └── __init__.py ├── cli2 │ └── __init__.py ├── cli_script.py ├── path.py ├── py.typed ├── types.py ├── utilities.py ├── v1 │ ├── AsyncClient.py │ ├── AsyncExternalStorage.py │ ├── BasicClient.py │ ├── ExternalStorage.py │ └── __init__.py └── v2 │ ├── __init__.py │ ├── _async │ ├── Client.py │ └── __init__.py │ └── _sync │ └── Client.py ├── pyproject.toml ├── tests ├── common.py ├── context.py ├── test_authorisation.py ├── test_compute.py ├── test_compute_async.py ├── test_extras.py ├── test_extras_async.py ├── test_status.py ├── test_status_async.py ├── test_storage.py ├── test_storage_async.py ├── test_utilities.py ├── test_utilities_async.py └── v2 │ ├── common.py │ ├── context_v2.py │ ├── handlers.py │ ├── responses │ ├── checksum.json │ ├── chmod.json │ ├── chown.json │ ├── chown_not_permitted.json │ ├── compress.json │ ├── delete.json │ ├── extract.json │ ├── file.json │ ├── head.json │ ├── head_bytes.json │ ├── head_bytes_exclude_trailing.json │ ├── head_lines.json │ ├── head_lines_exclude_trailing.json │ ├── job_info.json │ ├── job_metadata.json │ ├── job_submit.json │ ├── ls.json │ ├── ls_dereference.json │ ├── ls_hidden.json │ ├── ls_home.json │ ├── ls_home_dereference.json │ ├── ls_home_hidden.json │ ├── ls_home_recursive.json │ ├── ls_home_uid.json │ ├── ls_invalid_path.json │ ├── ls_recursive.json │ ├── ls_uid.json │ ├── mkdir.json │ ├── nodes.json │ ├── partitions.json │ ├── reservations.json │ ├── rm.json │ ├── stat.json │ ├── stat_dereference.json │ ├── symlink.json │ ├── systems.json │ ├── tail.json │ ├── tail_bytes.json │ ├── tail_bytes_exclude_beginning.json │ ├── tail_lines.json │ ├── tail_lines_exclude_beginning.json │ ├── userinfo.json │ └── view.json │ ├── test_v2_async.py │ ├── test_v2_cli.py │ └── test_v2_sync.py ├── tox.ini └── utils └── run_unasync.py /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | workflow_dispatch: 5 | release: 6 | types: [prereleased, released] 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | deploy: 13 | 14 | runs-on: ubuntu-22.04 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | - name: Set up Python 19 | uses: actions/setup-python@v3 20 | with: 21 | python-version: '3.8' 22 | - name: install flit 23 | run: | 24 | pip install flit~=3.4 25 | - name: Build and publish 26 | run: | 27 | flit publish 28 | env: 29 | FLIT_USERNAME: __token__ 30 | FLIT_PASSWORD: ${{ secrets.PYPI_TOKEN }} 31 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | name: Unittests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | 8 | jobs: 9 | unittest: 10 | 11 | runs-on: ubuntu-22.04 12 | strategy: 13 | matrix: 14 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install .[test] 26 | - name: Test with pytest 27 | working-directory: ./tests 28 | run: | 29 | pytest -v 30 | 31 | lint: 32 | 33 | runs-on: ubuntu-22.04 34 | 35 | steps: 36 | - uses: actions/checkout@v2 37 | - name: Set up Python 38 | uses: actions/setup-python@v2 39 | with: 40 | python-version: "3.8" 41 | - name: Install dependencies 42 | run: | 43 | python -m pip install --upgrade pip 44 | pip install .[test] 45 | - name: Lint with flake8 46 | run: | 47 | # stop the build if there are Python syntax errors or undefined names 48 | flake8 . --count --select=E9,F63,F7,F82,F821 --show-source --statistics 49 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 50 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 51 | 52 | type-check: 53 | 54 | runs-on: ubuntu-22.04 55 | 56 | steps: 57 | - uses: actions/checkout@v2 58 | - name: Set up Python 59 | uses: actions/setup-python@v2 60 | with: 61 | python-version: "3.8" 62 | - name: Install dependencies 63 | run: | 64 | python -m pip install --upgrade pip 65 | pip install .[test] 66 | - name: type check with mypy 67 | run: | 68 | mypy firecrest 69 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .DS_Store 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | #poetry.lock 104 | 105 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 106 | __pypackages__/ 107 | 108 | # Celery stuff 109 | celerybeat-schedule 110 | celerybeat.pid 111 | 112 | # SageMath parsed files 113 | *.sage.py 114 | 115 | # Environments 116 | .env 117 | .venv 118 | env/ 119 | venv/ 120 | ENV/ 121 | env.bak/ 122 | venv.bak/ 123 | 124 | # Spyder project settings 125 | .spyderproject 126 | .spyproject 127 | 128 | # Rope project settings 129 | .ropeproject 130 | 131 | # mkdocs documentation 132 | /site 133 | 134 | # mypy 135 | .mypy_cache/ 136 | .dmypy.json 137 | dmypy.json 138 | 139 | # Pyre type checker 140 | .pyre/ 141 | 142 | # pytype static type analyzer 143 | .pytype/ 144 | 145 | # Cython debug symbols 146 | cython_debug/ 147 | 148 | # PyCharm 149 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 150 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 151 | # and can be added to the global gitignore or merged into this file. For a more nuclear 152 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 153 | #.idea/ 154 | -------------------------------------------------------------------------------- /.mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy-unasync] 2 | ignore_missing_imports = True 3 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | version: 2 5 | 6 | build: 7 | os: "ubuntu-22.04" 8 | tools: 9 | python: "3.8" 10 | 11 | python: 12 | install: 13 | - method: pip 14 | path: . 15 | extra_requirements: 16 | - docs 17 | 18 | sphinx: 19 | configuration: docs/source/conf.py 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018-2023 ETH Zurich. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, 4 | are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 2. Redistributions in binary form must reproduce the above copyright notice, 9 | this list of conditions and the following disclaimer in the documentation 10 | and/or other materials provided with the distribution. 11 | 3. Neither the name of the copyright holder nor the names of its 12 | contributors may be used to endorse or promote products derived from this 13 | software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 19 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 22 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyFirecREST 2 | 3 | This is a simple python wrapper for the [FirecREST API](https://github.com/eth-cscs/firecrest). 4 | 5 | ### How to install 6 | - Through [PyPI](https://pypi.org/project/pyfirecrest/): 7 | 8 | ``` 9 | python3 -m pip install pyfirecrest 10 | ``` 11 | 12 | ### How to use it as a python package 13 | The full documentation of pyFirecREST is in [this page](https://pyfirecrest.readthedocs.io) but you can get an idea from the following example. 14 | This is how you can use the testbuild from the demo environment [here](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo). 15 | The configuration corresponds to the account `firecrest-sample`. 16 | 17 | ```python 18 | import firecrest as f7t 19 | 20 | # Configuration parameters for the Authorization Object 21 | client_id = "firecrest-sample" 22 | client_secret = "b391e177-fa50-4987-beaf-e6d33ca93571" 23 | token_uri = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token" 24 | 25 | # Create an authorization object with Client Credentials authorization grant 26 | keycloak = f7t.ClientCredentialsAuth( 27 | client_id, client_secret, token_uri 28 | ) 29 | 30 | # Setup the client for the specific account 31 | client = f7t.v1.Firecrest( 32 | firecrest_url="http://localhost:8000", authorization=keycloak 33 | ) 34 | 35 | try: 36 | parameters = client.parameters() 37 | print(f"Firecrest parameters: {parameters}") 38 | except f7t.FirecrestException as e: 39 | # When the error comes from the responses to a firecrest request you will get a 40 | # `FirecrestException` and from this you can examine the http responses yourself 41 | # through the `responses` property 42 | print(e) 43 | print(e.responses) 44 | except Exception as e: 45 | # You might also get regular exceptions in some cases. For example when you are 46 | # trying to upload a file that doesn't exist in your local filesystem. 47 | pass 48 | ``` 49 | 50 | ### How to use it from the terminal 51 | 52 | After version 1.3.0 pyFirecREST comes together with a CLI but for now it can only be used with the `f7t.ClientCredentialsAuth` authentication class. 53 | 54 | Assuming you are using the same client, you can start by setting as environment variables: 55 | ```bash 56 | export FIRECREST_CLIENT_ID=firecrest-sample 57 | export FIRECREST_CLIENT_SECRET=b391e177-fa50-4987-beaf-e6d33ca93571 58 | export FIRECREST_URL=http://localhost:8000 59 | export AUTH_TOKEN_URL=http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token 60 | ``` 61 | 62 | After that you can explore the capabilities of the CLI with the `--help` option: 63 | ```bash 64 | firecrest --help 65 | firecrest ls --help 66 | firecrest submit --help 67 | firecrest upload --help 68 | firecrest download --help 69 | firecrest submit-template --help 70 | ``` 71 | 72 | Some basic examples: 73 | ```bash 74 | # Get the parameters of different microservices of FirecREST 75 | firecrest parameters 76 | 77 | # Get the available systems 78 | firecrest systems 79 | 80 | # Set the environment variable to specify the name of the system 81 | export FIRECREST_SYSTEM="cluster" 82 | 83 | # List files of directory 84 | firecrest ls /home 85 | 86 | # Submit a job 87 | firecrest submit script.sh 88 | 89 | # Upload a "small" file (you can check the maximum size in `UTILITIES_MAX_FILE_SIZE` from the `parameters` command) 90 | firecrest upload --type=direct local_file.txt /path/to/cluster/fs 91 | 92 | # Upload a "large" file 93 | firecrest upload --type=external local_file.txt /path/to/cluster/fs 94 | # You will have to finish the upload with a second command that will be given in the output 95 | ``` 96 | -------------------------------------------------------------------------------- /async-example.md: -------------------------------------------------------------------------------- 1 | # Examples for asyncio with pyfirecrest 2 | 3 | ### Simple asynchronous workflow with the new client 4 | 5 | Here is an example of how to use the `AsyncFirecrest` client with asyncio. 6 | 7 | ```python 8 | import firecrest 9 | import asyncio 10 | import logging 11 | 12 | 13 | # Setup variables before running the script 14 | client_id = "" 15 | client_secret = "" 16 | token_uri = "" 17 | firecrest_url = "" 18 | 19 | machine = "" 20 | local_script_path = "" 21 | 22 | # Ignore this part, it is simply setup for logging 23 | logger = logging.getLogger("simple_example") 24 | logger.setLevel(logging.DEBUG) 25 | ch = logging.StreamHandler() 26 | ch.setLevel(logging.DEBUG) 27 | formatter = logging.Formatter("%(asctime)s - %(message)s", datefmt="%H:%M:%S") 28 | ch.setFormatter(formatter) 29 | logger.addHandler(ch) 30 | 31 | async def workflow(client, i): 32 | logger.info(f"{i}: Starting workflow") 33 | job = await client.submit(machine, local_script_path) 34 | logger.info(f"{i}: Submitted job with jobid: {job['jobid']}") 35 | while True: 36 | poll_res = await client.poll_active(machine, [job["jobid"]]) 37 | if len(poll_res) < 1: 38 | logger.info(f"{i}: Job {job['jobid']} is no longer active") 39 | break 40 | 41 | logger.info(f"{i}: Job {job['jobid']} status: {poll_res[0]['state']}") 42 | await asyncio.sleep(30) 43 | 44 | output = await client.view(machine, job["job_file_out"]) 45 | logger.info(f"{i}: job output: {output}") 46 | 47 | 48 | async def main(): 49 | auth = firecrest.ClientCredentialsAuth(client_id, client_secret, token_uri) 50 | client = firecrest.v1.AsyncFirecrest(firecrest_url, authorization=auth) 51 | 52 | # Set up the desired polling rate for each microservice. The float number 53 | # represents the number of seconds between consecutive requests in each 54 | # microservice. 55 | client.time_between_calls = { 56 | "compute": 1, 57 | "reservations": 0.5, 58 | "status": 0.5, 59 | "storage": 0.5, 60 | "tasks": 0.5, 61 | "utilities": 0.5, 62 | } 63 | 64 | workflows = [workflow(client, i) for i in range(5)] 65 | await asyncio.gather(*workflows) 66 | 67 | 68 | asyncio.run(main()) 69 | 70 | ``` 71 | 72 | 73 | ### External transfers with `AsyncFirecrest` 74 | 75 | The uploads and downloads work as before but you have to keep in mind which methods are coroutines. 76 | 77 | ```python 78 | # Download 79 | down_obj = await client.external_download("cluster", "/remote/path/to/the/file") 80 | status = await down_obj.status 81 | print(status) 82 | await down_obj.finish_download("my_local_file") 83 | 84 | # Upload 85 | up_obj = await client.external_upload("cluster", "/path/to/local/file", "/remote/path/to/filesystem") 86 | await up_obj.finish_upload() 87 | status = await up_obj.status 88 | print(status) 89 | ``` 90 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/source/authorization.rst: -------------------------------------------------------------------------------- 1 | Authorization 2 | ============= 3 | 4 | For every request to FirecREST you need to have a valid access token. 5 | This will enable your application to have access to the requested resources. 6 | 7 | 8 | .. Supported grant types 9 | .. --------------------- 10 | 11 | .. Implicit 12 | .. ^^^^^^^^ 13 | 14 | .. Client Credentials 15 | .. ^^^^^^^^^^^^^^^^^^ 16 | 17 | The pyFirecREST Authorization Object 18 | ------------------------------------ 19 | 20 | You can take care of the access token by yourself any way you want, or even better use a python library to take care of this for you, depending on the grant type. 21 | What pyFirecREST will need in the end is only a python object with the method ``get_access_token()``, that when called will provide a valid access token. 22 | 23 | Let's say for example you have somehow obtained a long-lasting access token. 24 | The Authorization class you would need to make and give to Firecrest would look like this: 25 | 26 | .. code-block:: Python 27 | 28 | class MyAuthorizationClass: 29 | def __init__(self): 30 | pass 31 | 32 | def get_access_token(self): 33 | return 34 | 35 | If this is your case you can move on to the next session, where you can see how to use this Authorization object, in order to make your requests. 36 | 37 | If you want to use the Client Credentials authorization grant, you can use the ``ClientCredentialsAuth`` class from pyFirecREST and setup the authorization object like this: 38 | 39 | .. code-block:: Python 40 | 41 | import firecrest as f7t 42 | 43 | keycloak = f7t.ClientCredentialsAuth( 44 | , , 45 | ) 46 | 47 | In a similar way you can reuse other packages to support different grant types, like `Flask-OIDC `__. 48 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('../..')) 16 | import firecrest 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'PyFirecREST' 21 | copyright = '2021-2023, CSCS Swiss National Supercomputing Center' 22 | author = 'CSCS Swiss National Supercomputing Center' 23 | release = firecrest.__version__ 24 | 25 | # -- General configuration --------------------------------------------------- 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = ['sphinx.ext.autodoc', 'sphinx_click'] 31 | 32 | # Add any paths that contain templates here, relative to this directory. 33 | templates_path = ['_templates'] 34 | 35 | # List of patterns, relative to source directory, that match files and 36 | # directories to ignore when looking for source files. 37 | # This pattern also affects html_static_path and html_extra_path. 38 | exclude_patterns = [] 39 | 40 | 41 | # -- Options for HTML output ------------------------------------------------- 42 | 43 | # The theme to use for HTML and HTML Help pages. See the documentation for 44 | # a list of builtin themes. 45 | # 46 | html_theme = 'sphinx_rtd_theme' 47 | 48 | # Add any paths that contain custom static files (such as style sheets) here, 49 | # relative to this directory. They are copied after the builtin static files, 50 | # so a file named "default.css" will overwrite the builtin "default.css". 51 | html_static_path = ['_static'] 52 | 53 | autodoc_member_order = 'bysource' 54 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Welcome to PyFirecREST 3 | ====================== 4 | 5 | PyFirecREST is a Python library to use the `FirecREST API `__. With it, you can manage your resources from Python 3 scripts. 6 | 7 | 8 | 9 | Installation 10 | ============ 11 | 12 | This package is in `PyPi `__, and ``python3 -m pip install pyfirecrest`` should be enough. 13 | You can also clone it from `Github `__ and even modify according to your needs. 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | :caption: Getting started: 18 | 19 | authorization 20 | tutorial_index 21 | 22 | 23 | .. toctree:: 24 | :maxdepth: 2 25 | :caption: Reference: 26 | 27 | reference_auth 28 | reference_v1_index 29 | reference_v2_index 30 | 31 | 32 | Contact 33 | ======= 34 | 35 | In case of questions/bugs/feature requests feel free to open issues in the public `repository in Github `__. 36 | -------------------------------------------------------------------------------- /docs/source/reference_async.rst: -------------------------------------------------------------------------------- 1 | Asynchronous FirecREST objects 2 | ============================== 3 | 4 | The library also provides an asynchronous API for the client: 5 | 6 | The ``AsyncFirecrest`` class 7 | **************************** 8 | .. autoclass:: firecrest.v1.AsyncFirecrest 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | 14 | The ``AsyncExternalDownload`` class 15 | *********************************** 16 | .. autoclass:: firecrest.v1.AsyncExternalDownload 17 | :inherited-members: 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | 23 | The ``AsyncExternalUpload`` class 24 | ********************************* 25 | .. autoclass:: firecrest.v1.AsyncExternalUpload 26 | :inherited-members: 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/reference_async_v2.rst: -------------------------------------------------------------------------------- 1 | Asynchronous FirecREST objects 2 | ============================== 3 | 4 | The library also provides an asynchronous API for the client: 5 | 6 | The ``AsyncFirecrest`` class 7 | **************************** 8 | .. autoclass:: firecrest.v2.AsyncFirecrest 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | The ``AsyncExternalUpload`` class 14 | ********************************* 15 | .. autoclass:: firecrest.v2.AsyncExternalUpload 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | The ``AsyncExternalDownload`` class 21 | *********************************** 22 | .. autoclass:: firecrest.v2.AsyncExternalDownload 23 | :members: 24 | :undoc-members: 25 | :show-inheritance: 26 | -------------------------------------------------------------------------------- /docs/source/reference_auth.rst: -------------------------------------------------------------------------------- 1 | Authorization 2 | ============= 3 | 4 | The ``ClientCredentialsAuth`` class 5 | *********************************** 6 | .. autoclass:: firecrest.ClientCredentialsAuth 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | -------------------------------------------------------------------------------- /docs/source/reference_basic.rst: -------------------------------------------------------------------------------- 1 | The basic client 2 | ================ 3 | 4 | The wrapper includes the ``Firecrest`` class, which is in practice a very basic client. 5 | Together with the authorisation class it takes care of the token and makes the appropriate calls for each action. 6 | 7 | The ``Firecrest`` class 8 | *********************** 9 | .. autoclass:: firecrest.v1.Firecrest 10 | :members: 11 | :undoc-members: 12 | :show-inheritance: 13 | 14 | 15 | The ``ExternalUpload`` class 16 | **************************** 17 | .. autoclass:: firecrest.v1.ExternalUpload 18 | :inherited-members: 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | The ``ExternalDownload`` class 25 | ****************************** 26 | .. autoclass:: firecrest.v1.ExternalDownload 27 | :inherited-members: 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | 32 | 33 | Custom types of the library 34 | *************************** 35 | .. automodule:: firecrest.types 36 | :members: 37 | :undoc-members: 38 | -------------------------------------------------------------------------------- /docs/source/reference_cli.rst: -------------------------------------------------------------------------------- 1 | Command-line Interface 2 | ====================== 3 | 4 | The best way to try the cli and see its capabilities is with the ``--help`` option in every subcommand. 5 | You can also find here a complete list of the subcommands, options and arguments. 6 | 7 | CLI reference 8 | ************* 9 | .. click:: firecrest.cli:typer_click_object 10 | :prog: firecrest 11 | :nested: full -------------------------------------------------------------------------------- /docs/source/reference_sync_v2.rst: -------------------------------------------------------------------------------- 1 | FirecREST objects 2 | ============================== 3 | 4 | Here is the API for the client: 5 | 6 | The ``Firecrest`` class 7 | **************************** 8 | .. autoclass:: firecrest.v2.Firecrest 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | The ``ExternalUpload`` class 14 | **************************** 15 | .. autoclass:: firecrest.v2.ExternalUpload 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | The ``ExternalDownload`` class 21 | ****************************** 22 | .. autoclass:: firecrest.v2.ExternalDownload 23 | :members: 24 | :undoc-members: 25 | :show-inheritance: 26 | -------------------------------------------------------------------------------- /docs/source/reference_v1_index.rst: -------------------------------------------------------------------------------- 1 | API v1 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | reference_basic 9 | reference_async 10 | reference_cli 11 | -------------------------------------------------------------------------------- /docs/source/reference_v2_index.rst: -------------------------------------------------------------------------------- 1 | API v2 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | reference_sync_v2 9 | reference_async_v2 10 | -------------------------------------------------------------------------------- /docs/source/tutorial_async.rst: -------------------------------------------------------------------------------- 1 | How to use the asynchronous API 2 | =============================== 3 | 4 | In this tutorial, we will explore the asynchronous API of the pyFirecREST library. 5 | Asynchronous programming is a powerful technique that allows you to write more efficient and responsive code by handling concurrent tasks without blocking the main execution flow. 6 | This capability is particularly valuable when dealing with time-consuming operations such as network requests, I/O operations, or interactions with external services. 7 | 8 | In order to take advantage of the asynchronous client you may need to make many changes in your existing code, so the effort is worth it when you develop a code from the start or if you need to make a large number of requests. 9 | You could submit hundreds or thousands of jobs, set a reasonable rate and pyFirecREST will handle it in the background without going over the request rate limit or overflowing the system. 10 | 11 | If you are already familiar with the synchronous version of pyFirecREST, you will find it quite straightforward to adapt to the asynchronous paradigm. 12 | 13 | We will be going through an example that will use the `asyncio library `__. 14 | First you will need to create an ``AsyncFirecrest`` object, instead of the simple ``Firecrest`` object. 15 | 16 | .. code-block:: Python 17 | 18 | client = fc.v1.AsyncFirecrest( 19 | firecrest_url=, 20 | authorization=MyAuthorizationClass() 21 | ) 22 | 23 | As you can see in the reference, the methods of ``AsyncFirecrest`` have the same name as the ones from the simple client, with the same arguments and types, but you will need to use the async/await syntax when you call them. 24 | 25 | Here is an example of the calls we saw in the previous section: 26 | 27 | .. code-block:: Python 28 | 29 | # Getting all the systems 30 | systems = await client.all_systems() 31 | print(systems) 32 | 33 | # Getting the files of a directory 34 | files = await client.list_files("cluster", "/home/test_user") 35 | print(files) 36 | 37 | # Submit a job 38 | job = await client.submit("cluster", script_local_path="script.sh") 39 | print(job) 40 | 41 | 42 | The uploads and downloads work as before but you have to keep in mind which methods are coroutines. 43 | 44 | .. code-block:: Python 45 | 46 | # Download 47 | down_obj = await client.external_download("cluster", "/remote/path/to/the/file") 48 | status = await down_obj.status 49 | print(status) 50 | await down_obj.finish_download("my_local_file") 51 | 52 | # Upload 53 | up_obj = await client.external_upload("cluster", "/path/to/local/file", "/remote/path/to/filesystem") 54 | await up_obj.finish_upload() 55 | status = await up_obj.status 56 | print(status) 57 | 58 | 59 | Here is a more complete example for how you could use the asynchronous client: 60 | 61 | 62 | .. code-block:: Python 63 | 64 | import firecrest 65 | import asyncio 66 | import logging 67 | 68 | 69 | # Setup variables before running the script 70 | client_id = "" 71 | client_secret = "" 72 | token_uri = "" 73 | firecrest_url = "" 74 | 75 | machine = "" 76 | local_script_path = "" 77 | 78 | # This is simply setup for logging, you can ignore it 79 | logger = logging.getLogger("simple_example") 80 | logger.setLevel(logging.DEBUG) 81 | ch = logging.StreamHandler() 82 | ch.setLevel(logging.DEBUG) 83 | formatter = logging.Formatter("%(asctime)s - %(message)s", datefmt="%H:%M:%S") 84 | ch.setFormatter(formatter) 85 | logger.addHandler(ch) 86 | 87 | async def workflow(client, i): 88 | logger.info(f"{i}: Starting workflow") 89 | job = await client.submit(machine, script_local_path=local_script_path) 90 | logger.info(f"{i}: Submitted job with jobid: {job['jobid']}") 91 | while True: 92 | poll_res = await client.poll_active(machine, [job["jobid"]]) 93 | if len(poll_res) < 1: 94 | logger.info(f"{i}: Job {job['jobid']} is no longer active") 95 | break 96 | 97 | logger.info(f"{i}: Job {job['jobid']} status: {poll_res[0]['state']}") 98 | await asyncio.sleep(30) 99 | 100 | output = await client.view(machine, job["job_file_out"]) 101 | logger.info(f"{i}: job output: {output}") 102 | 103 | 104 | async def main(): 105 | auth = firecrest.ClientCredentialsAuth(client_id, client_secret, token_uri) 106 | client = firecrest.AsyncFirecrest(firecrest_url, authorization=auth) 107 | 108 | # Set up the desired polling rate for each microservice. The float number 109 | # represents the number of seconds between consecutive requests in each 110 | # microservice. 111 | client.time_between_calls = { 112 | "compute": 5, 113 | "reservations": 5, 114 | "status": 5, 115 | "storage": 5, 116 | "tasks": 5, 117 | "utilities": 5, 118 | } 119 | 120 | workflows = [workflow(client, i) for i in range(5)] 121 | await asyncio.gather(*workflows) 122 | 123 | 124 | asyncio.run(main()) 125 | -------------------------------------------------------------------------------- /docs/source/tutorial_basic_v1.rst: -------------------------------------------------------------------------------- 1 | Tutorial for FirecREST v1 2 | ========================= 3 | 4 | Your starting point to use pyFirecREST will be the creation of a FirecREST object. 5 | This is simply a mini client that, in cooperation with the authorization object, will take care of the necessary requests that need to be made and handle the responses. 6 | 7 | If you want to understand how to setup your authorization object have a look at the previous section. 8 | For this tutorial we will assume the simplest kind of authorization class, where the same token will always be used. 9 | 10 | .. code-block:: Python 11 | 12 | import firecrest as fc 13 | 14 | class MyAuthorizationClass: 15 | def __init__(self): 16 | pass 17 | 18 | def get_access_token(self): 19 | return 20 | 21 | # Setup the client with the appropriate URL and the authorization class 22 | client = fc.v1.Firecrest(firecrest_url=, authorization=MyAuthorizationClass()) 23 | 24 | 25 | Simple blocking requests 26 | ------------------------ 27 | 28 | Most of the methods of the FirecREST object require a simple http request to FirecREST. 29 | With the client we just created here are a couple of examples of listing the files of a directory or getting all the available systems of FirecREST. 30 | 31 | 32 | Getting all the available systems 33 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 34 | 35 | A good start, to make sure your token is valid, is to get the names of all the available systems, where FirecREST can give you access. 36 | This will definitely be useful in the future. 37 | 38 | .. code-block:: Python 39 | 40 | systems = client.all_systems() 41 | print(systems) 42 | 43 | Systems is going to be a list of systems and their properties, and you will have to choose from one of them. 44 | This is an example of the output: 45 | 46 | .. code-block:: json 47 | 48 | [ 49 | { 50 | "description": "System ready", 51 | "status": "available", 52 | "system": "cluster" 53 | }, 54 | { 55 | "description": "System ready", 56 | "status": "available", 57 | "system": "cluster2" 58 | } 59 | ] 60 | 61 | Listing files in a directory 62 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 63 | 64 | Let's say you want to list the directory in the filesystem of a machine called "cluster". 65 | You can get a list of the files, with all the usual properties that ls provides (size, type, permissions etc). 66 | 67 | .. code-block:: Python 68 | 69 | files = client.list_files("cluster", "/home/test_user") 70 | print(files) 71 | 72 | The output will be something like this: 73 | 74 | .. code-block:: json 75 | 76 | [ 77 | { 78 | "group": "test_user", 79 | "last_modified": "2020-04-11T14:53:11", 80 | "link_target": "", 81 | "name": "test_directory", 82 | "permissions": "rwxrwxr-x", 83 | "size": "4096", 84 | "type": "d", 85 | "user": "test_user" 86 | }, 87 | { 88 | "group": "test_user", 89 | "last_modified": "2020-04-11T14:14:23", 90 | "link_target": "", 91 | "name": "test_file.txt", 92 | "permissions": "rw-rw-r--", 93 | "size": "10", 94 | "type": "-", 95 | "user": "test_user" 96 | } 97 | ] 98 | 99 | Interact with the scheduler 100 | --------------------------- 101 | 102 | pyFirecREST offers three basic functionalities of the scheduler: submit jobs on behalf of a user, poll for the jobs of the user and cancel jobs. 103 | Although the methods of this client will be blocking, on the background it will make at least two requests to Firecrest to return the results of the action. 104 | 105 | This is how can make a simple job submission, when the batch script is on your local filesystem: 106 | 107 | .. code-block:: Python 108 | 109 | job = client.submit("cluster", script_local_path="script.sh") 110 | print(job) 111 | 112 | For a successful submission the output would look like this. 113 | 114 | .. code-block:: json 115 | 116 | { 117 | "job_data_err": "", 118 | "job_data_out": "", 119 | "job_file": "/home/test_user/firecrest/cfd276f40d7ee4f9d082b73b29a4d76e/script.sh", 120 | "job_file_err": "/home/test_user/firecrest/cfd276f40d7ee4f9d082b73b29a4d76e/slurm-2.out", 121 | "job_file_out": "/home/test_user/firecrest/cfd276f40d7ee4f9d082b73b29a4d76e/slurm-2.out", 122 | "jobid": 42, 123 | "result": "Job submitted" 124 | } 125 | 126 | From the returned fields, you can see the result of the job submission (``result``), the ``jobid``, the location of the uploaded batch script (``job_file``) as well as the location of output (``job_file_out``) and error (``job_file_err``) files. Finally you also get the content of the beginning output and error file, but since the job probably hasn't started running yet, it will be empty. 127 | 128 | All requests that involve the scheduler will create a FirecREST task and be part of an internal queue. 129 | When you upload a batch script, FirecREST will create a new directory called ``firecrest``, and a subdirectory there with the Firecrest task ID. It will upload the batchscript there and submit the job from this directory. 130 | 131 | If you choose to submit the job with a batch script in the machine's filesystem, with the option ``script_remote_path=/path/to/script``, then FirecREST will submit the job from the directory of the batch script. 132 | 133 | This method hides the multiple requests and will be blocking, but you can find more information about the job submission `here `__. 134 | 135 | Transfer of large files 136 | ----------------------- 137 | 138 | For larger files the user cannot directly upload/download a file to/from FirecREST. 139 | A staging area will be used and the process will require multiple requests from the user. 140 | 141 | External Download 142 | ^^^^^^^^^^^^^^^^^ 143 | 144 | For example in the external download process, the requested file will first have to be moved to the staging area. 145 | **This could take a long time in case of a large file.** 146 | When this process finishes, FirecREST will have created a dedicated space for this file and the user can download the file locally as many times as he wants. 147 | You can follow this process with the status codes of the task: 148 | 149 | +--------+--------------------------------------------------------------------+ 150 | | Status | Description | 151 | +========+====================================================================+ 152 | | 116 | Started upload from filesystem to Object Storage | 153 | +--------+--------------------------------------------------------------------+ 154 | | 117 | Upload from filesystem to Object Storage has finished successfully | 155 | +--------+--------------------------------------------------------------------+ 156 | | 118 | Upload from filesystem to Object Storage has finished with errors | 157 | +--------+--------------------------------------------------------------------+ 158 | 159 | In code it would look like this: 160 | 161 | .. code-block:: Python 162 | 163 | # This call will only start the transfer of the file to the staging area 164 | down_obj = client.external_download("cluster", "/remote/path/to/the/file") 165 | 166 | # You can follow the progress of the transfer through the status property 167 | print(down_obj.status) 168 | 169 | # As soon as down_obj.status is 117 we can proceed with the download to a local file 170 | down_obj.finish_download("my_local_file") 171 | 172 | You can download the file as many times as you want from the staging area. 173 | In case you want to get directly the link in the staging area you can call ``object_storage_data`` and finish the download in your prefered way. 174 | 175 | The methods ``finish_download`` and ``object_storage_data`` are blocking, and they will keep making requests to FirecREST until the status of the task is ``117`` or ``118``. 176 | You could also use the ``status`` property of the object to poll with your prefered rate for task progress, before calling them. 177 | 178 | Finally, when you finish your download it would be more safe to invalidate the link to the staging area, with the ``invalidate_object_storage_link`` method. 179 | 180 | External Upload 181 | ^^^^^^^^^^^^^^^ 182 | 183 | The case of external upload is very similar. 184 | To upload a file you would have to ask for the link in the staging area and upload the file there. 185 | **Even after uploading the file there, it will take some time for the file to appear in the filesystem.** 186 | You can alway follow the status of the task with the ``status`` method and when the file has been successfully uploaded the status of the task will be 114. 187 | 188 | +--------+--------------------------------------------------------------------+ 189 | | Status | Description | 190 | +========+====================================================================+ 191 | | 110 | Waiting for Form URL from Object Storage to be retrieved | 192 | +--------+--------------------------------------------------------------------+ 193 | | 111 | Form URL from Object Storage received | 194 | +--------+--------------------------------------------------------------------+ 195 | | 112 | Object Storage confirms that upload to Object Storage has finished | 196 | +--------+--------------------------------------------------------------------+ 197 | | 113 | Download from Object Storage to server has started | 198 | +--------+--------------------------------------------------------------------+ 199 | | 114 | Download from Object Storage to server has finished | 200 | +--------+--------------------------------------------------------------------+ 201 | | 115 | Download from Object Storage error | 202 | +--------+--------------------------------------------------------------------+ 203 | 204 | The simplest way to do the uploading through pyFirecREST is as follows: 205 | 206 | .. code-block:: Python 207 | 208 | # This call will only create the link to Object Storage 209 | up_obj = client.external_upload("cluster", "/path/to/local/file", "/remote/path/to/filesystem") 210 | 211 | # As soon as up_obj.status is 111 we can proceed with the upload of local file to the staging area 212 | up_obj.finish_upload() 213 | 214 | # You can follow the progress of the transfer through the status property 215 | print(up_obj.status) 216 | 217 | But, as before, you can get the necessary components for the upload from the ``object_storage_data`` property. 218 | You can get the link, as well as all the necessary arguments for the request to Object Storage and the full command you could perform manually from the terminal. 219 | -------------------------------------------------------------------------------- /docs/source/tutorial_basic_v2.rst: -------------------------------------------------------------------------------- 1 | Tutorial for FirecREST v2 2 | ========================= 3 | 4 | This tutorial will guide you through the basic functionalities of v2 of the FirecREST API. 5 | Since the API of FirecREST v2 has some important differences, the python client cannot be the same as the one for FirecREST v1. 6 | 7 | Your starting point to use pyFirecREST will be the creation of a `FirecREST` object. 8 | This mini client works with the authorization object to manage necessary requests and handle responses efficiently. 9 | 10 | If you want to understand how to setup your authorization object have a look at the previous section. 11 | For this tutorial we will assume the simplest kind of authorization class, where the same token will always be used. 12 | 13 | .. code-block:: Python 14 | 15 | import firecrest as fc 16 | 17 | class MyAuthorizationClass: 18 | def __init__(self): 19 | pass 20 | 21 | def get_access_token(self): 22 | return 23 | 24 | # Setup the client with the appropriate URL and the authorization class 25 | client = fc.v2.Firecrest(firecrest_url=, authorization=MyAuthorizationClass()) 26 | 27 | 28 | Simple blocking requests 29 | ------------------------ 30 | 31 | Most of the methods of the FirecREST object require a simple http request to FirecREST. 32 | With the client we just created here are a couple of examples of listing the files of a directory or getting all the available systems of FirecREST. 33 | 34 | 35 | Getting all the available systems 36 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 37 | 38 | A good starting point is to retrieve the list of systems available in FirecREST. This validates your token and helps you choose a target system for future requests. 39 | 40 | .. code-block:: Python 41 | 42 | systems = client.systems() 43 | print(systems) 44 | 45 | Systems is going to be a list of systems and their properties, and you will have to choose from one of them. 46 | This is an example of the output: 47 | 48 | .. code-block:: json 49 | 50 | [ 51 | { 52 | "name": "cluster", 53 | "host": "cluster.alps.cscs.ch", 54 | "sshPort": 22, 55 | "sshCertEmbeddedCmd": true, 56 | "scheduler": { 57 | "type": "slurm", 58 | "version": "24.05.4", 59 | "apiUrl": null, 60 | "apiVersion": null 61 | }, 62 | "servicesHealth": [ 63 | { 64 | "serviceType": "scheduler", 65 | "lastChecked": "2025-01-06T11:09:29.975235Z", 66 | "latency": 0.6163430213928223, 67 | "healthy": true, 68 | "message": null, 69 | "nodes": { 70 | "available": 280, 71 | "total": 600 72 | } 73 | }, 74 | { 75 | "serviceType": "ssh", 76 | "lastChecked": "2025-01-06T11:09:29.951104Z", 77 | "latency": 0.5919253826141357, 78 | "healthy": true, 79 | "message": null 80 | }, 81 | { 82 | "serviceType": "filesystem", 83 | "lastChecked": "2025-01-06T11:09:29.955848Z", 84 | "latency": 0.5964689254760742, 85 | "healthy": true, 86 | "message": null, 87 | "path": "/capstor/scratch/cscs" 88 | }, 89 | { 90 | "serviceType": "filesystem", 91 | "lastChecked": "2025-01-06T11:09:29.955997Z", 92 | "latency": 0.59639573097229, 93 | "healthy": true, 94 | "message": null, 95 | "path": "/users" 96 | }, 97 | { 98 | "serviceType": "filesystem", 99 | "lastChecked": "2025-01-06T11:09:29.955792Z", 100 | "latency": 0.5958302021026611, 101 | "healthy": true, 102 | "message": null, 103 | "path": "/capstor/store/cscs" 104 | } 105 | ], 106 | "probing": { 107 | "interval": 300, 108 | "timeout": 10, 109 | "maxLatency": null, 110 | "maxLoad": null 111 | }, 112 | "fileSystems": [ 113 | { 114 | "path": "/capstor/scratch/cscs", 115 | "dataType": "scratch", 116 | "defaultWorkDir": true 117 | }, 118 | { 119 | "path": "/users", 120 | "dataType": "users", 121 | "defaultWorkDir": false 122 | }, 123 | { 124 | "path": "/capstor/store/cscs", 125 | "dataType": "store", 126 | "defaultWorkDir": false 127 | } 128 | ], 129 | "datatransferJobsDirectives": [ 130 | "#SBATCH --nodes=1", 131 | "#SBATCH --time=0-00:15:00" 132 | ], 133 | "timeouts": { 134 | "sshConnection": 5, 135 | "sshLogin": 5, 136 | "sshCommandExecution": 5 137 | } 138 | } 139 | ] 140 | 141 | Listing files in a directory 142 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 143 | 144 | Let's say you want to list the directory in the filesystem of a machine called "cluster". 145 | You can get a list of the files, with all the usual properties that ls provides (size, type, permissions etc). 146 | 147 | .. code-block:: Python 148 | 149 | files = client.list_files("cluster", "/home/test_user") 150 | print(files) 151 | 152 | The output will be something like this: 153 | 154 | .. code-block:: json 155 | 156 | [ 157 | { 158 | "group": "test_user", 159 | "lastModified": "2020-04-11T14:53:11", 160 | "linkTarget": "", 161 | "name": "test_directory", 162 | "permissions": "rwxrwxr-x", 163 | "size": "4096", 164 | "type": "d", 165 | "user": "test_user" 166 | }, 167 | { 168 | "group": "test_user", 169 | "lastModified": "2020-04-11T14:14:23", 170 | "linkTarget": "", 171 | "name": "test_file.txt", 172 | "permissions": "rw-rw-r--", 173 | "size": "10", 174 | "type": "-", 175 | "user": "test_user" 176 | } 177 | ] 178 | 179 | Interact with the scheduler 180 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ 181 | 182 | FirecREST v2 simplifies job submission, monitoring, and cancellation. These operations now require only a single API request. 183 | As a result the pyFirecREST client has been simplified and the user can interact with the scheduler in a more efficient way. 184 | 185 | This is how can make a simple job submission, when the batch script is on your local filesystem: 186 | 187 | .. code-block:: Python 188 | 189 | job = client.submit("cluster", working_directory="/home/test_user", script_local_path="script.sh") 190 | print(job) 191 | 192 | For a successful submission the output would look like this. 193 | 194 | .. code-block:: json 195 | 196 | { 197 | "jobid": 42, 198 | } 199 | 200 | In FirecREST v2, the user selects the working directory where the job will be submitted from. 201 | 202 | Transfer of files 203 | ----------------- 204 | 205 | The two clients of FirecREST v2 have been designed to hide as much as possible the complexity of transferring large files. 206 | 207 | Internal transfers 208 | ^^^^^^^^^^^^^^^^^^ 209 | 210 | Copying, moving and removing files is done through scheduler jobs. 211 | The corresponding methods are `copy`, `move` and `remove` and will return a dictionary with information about the job. 212 | The client can either wait for the job to finish or not, in which case it lets the user handle it. 213 | In case the transfer is not successful, an exception will be raised and more details can be found in the log files of the job. 214 | 215 | External transfers 216 | ^^^^^^^^^^^^^^^^^^ 217 | 218 | Small files can be transfered directly to/from FirecREST, and will be immediately available to the user. 219 | Larger data transfers are handled by a job that will be submitted to the scheduler. 220 | The files need to be moved first to a staging area, before they are moved to the final directory. 221 | There is again the option to let the client handle the job submission or to do it manually. 222 | In case of small files the client will return ``None`` or raise an error if the transfer was not successful. 223 | For large files the client will return an object with information about the job and methods to finish the job in steps. 224 | 225 | Here is a simple example of how to transfer a file to a remote machine: 226 | 227 | .. code-block:: Python 228 | 229 | # If you want to easily download or upload a file you can use `blocking=True` 230 | # and let the client take care of the whole transfer 231 | client.download( 232 | system_name="cluster", 233 | source_path="/scratch/test_user/file.txt", 234 | target_path="/home/test_user/local_file.txt", 235 | account="scheduler_project", 236 | blocking=True 237 | ) 238 | 239 | If you want to do it in steps, you can do each step from the functions of ``ExternalDownload`` object or use your own custom functions. 240 | Here is the workflow broken down in steps: 241 | 242 | .. code-block:: Python 243 | 244 | download_obj = client.download( 245 | system_name="cluster", 246 | source_path="/scratch/test_user/file.txt", 247 | target_path="/home/test_user/local_file.txt", 248 | account="scheduler_project", 249 | blocking=False 250 | ) 251 | # For small files the download will return `None` and the file will be available in the target directory 252 | # For large files the download will return an object with information about the job 253 | if download_obj: 254 | print(download_obj.transfer_info) 255 | # You can also set an optional timeout for the job 256 | download_obj.wait_for_transfer_job() 257 | download_obj.download_file_from_stage() 258 | 259 | Similarly for the upload, you can use ``blocking=True`` 260 | 261 | .. code-block:: Python 262 | 263 | client.upload( 264 | system_name="cluster", 265 | local_file="/home/test_user/local_file.txt", 266 | directory="/scratch/test_user/", 267 | filename="file.txt", 268 | account="scheduler_project", 269 | blocking=True 270 | ) 271 | 272 | or do it in steps: 273 | 274 | .. code-block:: Python 275 | 276 | upload_obj = client.upload( 277 | system_name="cluster", 278 | local_file="/home/test_user/local_file.txt", 279 | directory="/scratch/test_user/", 280 | filename="file.txt", 281 | account="scheduler_project", 282 | blocking=False 283 | ) 284 | # For small files the upload will return `None` and the file will be directly 285 | # available in the target directory. 286 | # For large files the upload will return an object with information about the job. 287 | if upload_obj: 288 | print(upload_obj.transfer_info) 289 | upload_obj.upload_file_to_stage() 290 | # You can also set an optional timeout for the job 291 | upload_obj.wait_for_transfer_job() 292 | 293 | .. note:: 294 | 295 | If you are using the asynchronous version of the client, you simply need the ``await`` keyword in front of the ``upload``, ``download``, ``download_file_from_stage``, ``upload_file_to_stage`` and ``wait_for_transfer_job`` functions. 296 | Check the Reference section to find out which functions are asynchronous in the async client. 297 | -------------------------------------------------------------------------------- /docs/source/tutorial_cli.rst: -------------------------------------------------------------------------------- 1 | How to use the CLI 2 | ================== 3 | 4 | After version 1.3.0, pyFirecREST comes together with a CLI but for now it can only be used with the ``ClientCredentialsAuth`` authentication class. 5 | 6 | .. attention:: 7 | 8 | The CLI currently only supports FirecREST v1. Support for v2 is planned for the next release. 9 | 10 | You will need to set the environment variables ``FIRECREST_CLIENT_ID``, ``FIRECREST_CLIENT_SECRET`` and ``AUTH_TOKEN_URL`` to set up the Client Credentials client, as well as ``FIRECREST_URL`` with the URL for the FirecREST instance you are using. 11 | 12 | After that you can explore the capabilities of the CLI with the `--help` option: 13 | 14 | .. code-block:: bash 15 | 16 | firecrest --help 17 | firecrest ls --help 18 | firecrest submit --help 19 | firecrest upload --help 20 | firecrest download --help 21 | firecrest submit-template --help 22 | 23 | Some basic examples: 24 | 25 | .. code-block:: bash 26 | 27 | # Get the available systems 28 | firecrest systems 29 | 30 | # Set the environment variable to specify the name of the system 31 | export FIRECREST_SYSTEM=cluster1 32 | 33 | # Get the parameters of different microservices of FirecREST 34 | firecrest parameters 35 | 36 | # List files of directory 37 | firecrest ls /home 38 | 39 | # Submit a job 40 | firecrest submit script.sh 41 | 42 | # Upload a "small" file (you can check the maximum size in `UTILITIES_MAX_FILE_SIZE` from the `parameters` command) 43 | firecrest upload --type=direct local_file.txt /path/to/cluster/fs 44 | 45 | # Upload a "large" file 46 | firecrest upload --type=external local_file.txt /path/to/cluster/fs 47 | # You will have to finish the upload with a second command that will be given in the output 48 | -------------------------------------------------------------------------------- /docs/source/tutorial_errors.rst: -------------------------------------------------------------------------------- 1 | How to catch and debug errors 2 | ============================= 3 | 4 | The methods of the ``Firecrest``, ``ExternalUpload`` and ``ExternalDownload`` objects will raise exceptions in case something goes wrong. 5 | The same happens for their asynchronous counterparts. 6 | When the error comes from the response of some request pyFirecREST will raise ``FirecrestException``. 7 | In these cases you can manually examine all the responses from the requests in order to get more information, when the message is not informative enough. 8 | These responses are from the requests package of python and you can get all types of useful information from it, like the status code, the json response, the headers and more. 9 | Here is an example of the code that will handle those failures. 10 | 11 | .. code-block:: Python 12 | 13 | import firecrest as fc 14 | 15 | 16 | try: 17 | files = client.list_files("cluster", "/home/test_user") 18 | print(f"List of files: {files}") 19 | except fc.FirecrestException as e: 20 | # You can just print the exception to get more information about the type of error, 21 | # for example an invalid or expired token. 22 | print(e) 23 | # Or you can manually examine the responses. 24 | print(e.responses[-1]) 25 | print(e.responses[-1].status_code) 26 | print(e.responses[-1].body) 27 | except Exception as e: 28 | # You might also get regular exceptions in some cases. For example when you are 29 | # trying to upload a file that doesn't exist in your local filesystem. 30 | print(f"A different exception was encountered: {e}") 31 | -------------------------------------------------------------------------------- /docs/source/tutorial_index.rst: -------------------------------------------------------------------------------- 1 | Tutorials 2 | ========= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | tutorial_basic_v1 9 | tutorial_basic_v2 10 | tutorial_logging 11 | tutorial_errors 12 | tutorial_cli 13 | tutorial_async 14 | -------------------------------------------------------------------------------- /docs/source/tutorial_logging.rst: -------------------------------------------------------------------------------- 1 | 2 | Enable logging in your python code 3 | ================================== 4 | 5 | The simplest way to enable logging in your code would be to add this in the beginning of your file: 6 | 7 | .. code-block:: Python 8 | 9 | import logging 10 | 11 | logging.basicConfig( 12 | level=logging.INFO, 13 | format="%(levelname)s:%(name)s:%(message)s", 14 | ) 15 | 16 | pyFirecREST has all of it's messages in `INFO` level. If you want to avoid messages from other packages, you can do the following: 17 | 18 | .. code-block:: Python 19 | 20 | import logging 21 | 22 | logging.basicConfig( 23 | level=logging.WARNING, 24 | format="%(levelname)s:%(name)s:%(message)s", 25 | ) 26 | logging.getLogger("firecrest").setLevel(logging.INFO) 27 | -------------------------------------------------------------------------------- /firecrest/Authorization.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019-2023, ETH Zurich. All rights reserved. 3 | # 4 | # Please, refer to the LICENSE file in the root directory. 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | # 7 | import logging 8 | import requests 9 | import time 10 | 11 | import firecrest.FirecrestException as fe 12 | 13 | from datetime import datetime 14 | from requests.compat import json # type: ignore 15 | from typing import Optional, Tuple, Union 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | class ClientCredentialsAuth: 21 | """ 22 | Client Credentials Authorization class. 23 | 24 | :param client_id: name of the client as registered in the authorization server 25 | :type client_id: string 26 | :param client_secret: secret associated to the client 27 | :type client_secret: string 28 | :param token_uri: URI of the token request in the authorization server (e.g. https://auth.your-server.com/auth/realms/cscs/protocol/openid-connect/token) 29 | :type token_uri: string 30 | :param min_token_validity: reuse OIDC token until {min_token_validity} sec before the expiration time (by default 10). Since the token will be checked by different microservices, setting more time in min_token_validity will ensure that the token doesn't expire in the middle of the request. 31 | :type min_token_validity: float 32 | """ 33 | 34 | def __init__( 35 | self, 36 | client_id: str, 37 | client_secret: str, 38 | token_uri: str, 39 | min_token_validity: int = 10, 40 | ): 41 | self._client_id = client_id 42 | self._client_secret = client_secret 43 | self._token_uri = token_uri 44 | self._access_token = None 45 | self._token_expiration_ts = None 46 | self._min_token_validity = min_token_validity 47 | #: It will be passed to all the requests that will be made. 48 | #: How many seconds to wait for the server to send data before giving up. 49 | #: After that time a `requests.exceptions.Timeout` error will be raised. 50 | #: 51 | #: It can be a float or a tuple. More details here: https://requests.readthedocs.io. 52 | self.timeout: Optional[ 53 | Union[float, Tuple[float, float], Tuple[float, None]] 54 | ] = None 55 | #: Disable all logging from this authorization object. 56 | self.disable_client_logging: bool = False 57 | 58 | def _log(self, level: int, msg: str) -> None: 59 | """Log a message with the given level on the client logger. 60 | """ 61 | if not self.disable_client_logging: 62 | logger.log(level, msg) 63 | 64 | def get_access_token(self) -> str: 65 | """Returns an access token to be used for accessing resources. 66 | If the request fails the token will be None 67 | """ 68 | 69 | # Make sure that the access token has at least {min_token_validity} sec left before 70 | # it expires, otherwise make a new request 71 | if ( 72 | self._access_token 73 | and self._token_expiration_ts 74 | and time.time() <= (self._token_expiration_ts - self._min_token_validity) 75 | ): 76 | self._log( 77 | logging.INFO, 78 | f"Reusing token, will renew after {datetime.fromtimestamp(self._token_expiration_ts - self._min_token_validity)}" 79 | ) 80 | return self._access_token 81 | 82 | headers = {"Content-Type": "application/x-www-form-urlencoded"} 83 | data = { 84 | "grant_type": "client_credentials", 85 | "client_id": self._client_id, 86 | "client_secret": self._client_secret, 87 | } 88 | resp = requests.post( 89 | self._token_uri, headers=headers, data=data, timeout=self.timeout 90 | ) 91 | try: 92 | resp_json = resp.json() 93 | except json.JSONDecodeError: 94 | resp_json = "" 95 | 96 | if not resp.ok: 97 | self._log( 98 | logging.CRITICAL, 99 | f"Could not obtain token: {fe.ClientsCredentialsException([resp])}" 100 | ) 101 | raise fe.ClientsCredentialsException([resp]) 102 | 103 | self._access_token = resp_json["access_token"] 104 | self._token_expiration_ts = time.time() + resp_json["expires_in"] 105 | assert self._token_expiration_ts is not None 106 | self._log( 107 | logging.INFO, 108 | f"Token expires at {datetime.fromtimestamp(self._token_expiration_ts)}" 109 | ) 110 | return self._access_token 111 | -------------------------------------------------------------------------------- /firecrest/FirecrestException.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019-2023, ETH Zurich. All rights reserved. 3 | # 4 | # Please, refer to the LICENSE file in the root directory. 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | # 7 | import json 8 | 9 | 10 | ERROR_HEADERS = { 11 | "X-A-Directory", 12 | "X-Error", 13 | "X-Invalid-Path", 14 | "X-Machine-Does-Not-Exist", 15 | "X-Machine-Not-Available", 16 | "X-Not-A-Directory", 17 | "X-Not-Found", 18 | "X-Permission-Denied", 19 | "X-Timeout", 20 | } 21 | 22 | 23 | class FirecrestException(Exception): 24 | """Base class for exceptions raised when using PyFirecREST.""" 25 | 26 | def __init__(self, responses): 27 | super().__init__() 28 | self._responses = responses 29 | 30 | @property 31 | def responses(self): 32 | return self._responses 33 | 34 | def __str__(self): 35 | try: 36 | last_json_response = self._responses[-1].json() 37 | except json.decoder.JSONDecodeError: 38 | last_json_response = None 39 | 40 | return f"last request: {self._responses[-1].status_code} {last_json_response}" 41 | 42 | 43 | class NotFound(FirecrestException): 44 | """Exception raised by an invalid path""" 45 | 46 | def __str__(self): 47 | return f"{super().__str__()}: FirecREST endpoint not found" 48 | 49 | 50 | class UnauthorizedException(FirecrestException): 51 | """Exception raised by an unauthorized request""" 52 | 53 | def __str__(self): 54 | return f"{super().__str__()}: unauthorized request" 55 | 56 | 57 | class ClientsCredentialsException(FirecrestException): 58 | """Exception raised by the request to the authorization server""" 59 | 60 | def __str__(self): 61 | return f"{super().__str__()}: Client credentials error" 62 | 63 | 64 | class HeaderException(FirecrestException): 65 | """Exception raised by a request with an error header""" 66 | 67 | def __str__(self): 68 | s = f"{super().__str__()}: " 69 | for h in ERROR_HEADERS: 70 | if h in self._responses[-1].headers: 71 | s += self._responses[-1].headers[h] 72 | break 73 | 74 | return s 75 | 76 | 77 | class UnexpectedStatusException(FirecrestException): 78 | """Exception raised when a request gets an unexpected status""" 79 | 80 | def __init__(self, responses, expected_status_code): 81 | super().__init__(responses) 82 | self._expected_status_code = expected_status_code 83 | 84 | def __str__(self): 85 | return f"{super().__str__()}: expected status {self._expected_status_code}" 86 | 87 | 88 | class NoJSONException(FirecrestException): 89 | """Exception raised when JSON in not included in the response""" 90 | 91 | def __str__(self): 92 | return f"{super().__str__()}: JSON is not included in the response" 93 | 94 | 95 | class StorageDownloadException(FirecrestException): 96 | """Exception raised by a failed external download""" 97 | 98 | 99 | class StorageUploadException(FirecrestException): 100 | """Exception raised by a failed external upload""" 101 | 102 | 103 | class PollingIterException(Exception): 104 | """Exception raised when the polling iterator is exhausted""" 105 | 106 | def __init__(self, task_id): 107 | self._task_id = task_id 108 | 109 | def __str__(self): 110 | return ( 111 | f"polling iterator for task {self._task_id} " 112 | f"is exhausted. Update `polling_sleep_times` of the client " 113 | f"to increase the number of polling attempts." 114 | ) 115 | 116 | 117 | class TransferJobFailedException(Exception): 118 | """Exception raised when the polling iterator is exhausted""" 119 | 120 | def __init__(self, transfer_job_info, file_not_found=False): 121 | self._transfer_job_info = transfer_job_info 122 | self._file_not_found = file_not_found 123 | 124 | def __str__(self): 125 | if self._file_not_found: 126 | return ( 127 | f"Logs for transfer job not found. Maybe the job was " 128 | f"cancelled. Check the transfer job for more information: " 129 | f"{self._transfer_job_info['transferJob']}" 130 | ) 131 | 132 | return ( 133 | f"Transfer job failed. Check the log files for more " 134 | f"information: {self._transfer_job_info['transferJob']}" 135 | ) 136 | 137 | 138 | class TransferJobTimeoutException(TransferJobFailedException): 139 | """Exception when the transfer job exceeds the user-defined timeout""" 140 | 141 | def __str__(self): 142 | return ( 143 | f"Transfer job has exceeded the user-defined timeout. " 144 | f"Transfer job was cancelled: " 145 | f"{self._transfer_job_info['transferJob']}." 146 | ) 147 | 148 | 149 | class MultipartUploadException(Exception): 150 | """Exception raised when a multipart upload fails""" 151 | 152 | def __init__(self, transfer_job_info, msg=None): 153 | self._transfer_job_info = transfer_job_info 154 | self._msg = msg 155 | 156 | def __str__(self): 157 | ret = f"{self._msg}: " if self._msg else "" 158 | ret += ( 159 | f"Multipart upload failed. Transfer info: " 160 | f"({self._transfer_job_info})" 161 | ) 162 | return ret 163 | 164 | 165 | class NotImplementedOnAPIversion(Exception): 166 | """Exception raised when a feature is not developed yet for the current API version""" 167 | -------------------------------------------------------------------------------- /firecrest/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019-2023, ETH Zurich. All rights reserved. 3 | # 4 | # Please, refer to the LICENSE file in the root directory. 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | # 7 | import sys 8 | 9 | 10 | __version__ = "3.1.0" 11 | __app_name__ = "firecrest" 12 | MIN_PYTHON_VERSION = (3, 7, 0) 13 | 14 | # Check python version 15 | if sys.version_info[:3] < MIN_PYTHON_VERSION: 16 | sys.stderr.write( 17 | "Unsupported Python version: " 18 | "Python >= %d.%d.%d is required\n" % MIN_PYTHON_VERSION 19 | ) 20 | sys.exit(1) 21 | 22 | from . import v1, v2 23 | from firecrest.Authorization import ClientCredentialsAuth 24 | from firecrest.FirecrestException import ( 25 | ClientsCredentialsException, 26 | FirecrestException, 27 | HeaderException, 28 | NotImplementedOnAPIversion, 29 | StorageDownloadException, 30 | StorageUploadException, 31 | UnauthorizedException, 32 | UnexpectedStatusException, 33 | ) 34 | -------------------------------------------------------------------------------- /firecrest/cli_script.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2019-2023, ETH Zurich. All rights reserved. 4 | # 5 | # Please, refer to the LICENSE file in the root directory. 6 | # SPDX-License-Identifier: BSD-3-Clause 7 | # 8 | from firecrest import cli, __app_name__ 9 | from firecrest import cli2 10 | import os 11 | 12 | 13 | def main() -> None: 14 | # TODO: This is a temporary solution to support both API versions 15 | # in the same CLI script. We can have better support from the URL path or 16 | # the API response headers of v2. 17 | ver = os.environ.get("FIRECREST_API_VERSION") 18 | if ver and ver.startswith("1"): 19 | cli.app(prog_name=__app_name__) 20 | else: 21 | cli2.app(prog_name=__app_name__) 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /firecrest/py.typed: -------------------------------------------------------------------------------- 1 | # Marker file for PEP 561 2 | -------------------------------------------------------------------------------- /firecrest/types.py: -------------------------------------------------------------------------------- 1 | """Types returned by the API. 2 | 3 | See also: https://firecrest-api.cscs.ch 4 | """ 5 | from __future__ import annotations 6 | 7 | import sys 8 | from typing import Any 9 | 10 | if sys.version_info >= (3, 8): 11 | from typing import TypedDict 12 | else: 13 | from typing_extensions import TypedDict 14 | 15 | 16 | class Parameter(TypedDict): 17 | """A parameter record, from `status/parameters/{name}`""" 18 | 19 | name: str 20 | unit: str 21 | value: Any 22 | description: str 23 | 24 | 25 | class Parameters(TypedDict): 26 | """A parameters record, from `status/parameters`. For older versions 27 | of the API `compute` and `system` may not be present. 28 | """ 29 | 30 | storage: list[Parameter] 31 | utilities: list[Parameter] 32 | compute: list[Parameter] 33 | general: list[Parameter] 34 | 35 | 36 | class Service(TypedDict): 37 | """A service record, from `status/services/{name}`""" 38 | 39 | description: str 40 | service: str 41 | status: str 42 | status_code: int 43 | 44 | 45 | class System(TypedDict): 46 | """A system record, from `status/systems/{name}`""" 47 | 48 | description: str 49 | status: str 50 | system: str 51 | 52 | 53 | class Filesystem(TypedDict): 54 | """A filesystem record, from `status/filesystems`""" 55 | 56 | name: str 57 | path: str 58 | description: str 59 | status: int 60 | status_code: str 61 | 62 | 63 | class Task(TypedDict): 64 | """A task record, from `/tasks`""" 65 | 66 | created_at: str 67 | data: Any 68 | description: str 69 | hash_id: str 70 | last_modify: str 71 | service: str 72 | status: str 73 | system: str 74 | task_id: str 75 | updated_at: str 76 | user: str 77 | 78 | 79 | class LsFile(TypedDict): 80 | """A file listing record, from `utilities/ls`""" 81 | 82 | group: str 83 | last_modified: str 84 | link_target: str 85 | name: str 86 | permissions: str 87 | size: str 88 | type: str 89 | user: str 90 | 91 | 92 | class StatFile(TypedDict): 93 | """A file stat record, from `utilities/stat` 94 | 95 | Command is `stat {deref} -c '%a %i %d %h %u %g %s %X %Y %Z` 96 | 97 | See also https://docs.python.org/3/library/os.html#os.stat_result 98 | """ 99 | 100 | atime: int 101 | ctime: int 102 | dev: int # device 103 | gid: int # group id of owner 104 | ino: int # inode number 105 | mode: int # protection bits 106 | mtime: int 107 | nlink: int # number of hard links 108 | size: int # size of file, in bytes 109 | uid: int # user id of owner 110 | 111 | 112 | class JobAcct(TypedDict): 113 | """A job accounting record, from `compute/acct`""" 114 | 115 | jobid: str 116 | name: str 117 | nodelist: str 118 | nodes: str 119 | partition: str 120 | start_time: str 121 | state: str 122 | time: str 123 | time_left: str 124 | user: str 125 | 126 | 127 | class JobQueue(TypedDict): 128 | """A job queue record, from `compute/jobs`""" 129 | 130 | job_data_err: str 131 | job_data_out: str 132 | job_file: str 133 | job_file_err: str 134 | job_file_out: str 135 | jobid: str 136 | name: str 137 | nodelist: str 138 | nodes: str 139 | partition: str 140 | start_time: str 141 | state: str 142 | time: str 143 | time_left: str 144 | user: str 145 | 146 | 147 | class NodeInfo(TypedDict): 148 | """A node record record, from `compute/nodes`""" 149 | 150 | NodeName: str 151 | ActiveFeatures: list[str] 152 | Partitions: list[str] 153 | State: list[str] 154 | 155 | 156 | class PartitionInfo(TypedDict): 157 | """A node record record, from `compute/partitions`""" 158 | 159 | Default: str 160 | PartitionName: str 161 | State: str 162 | TotalCPUs: str 163 | TotalNodes: str 164 | 165 | 166 | class ReservationInfo(TypedDict): 167 | """A job queue record, from `compute/reservations`""" 168 | 169 | ReservationName: str 170 | State: str 171 | Nodes: str 172 | StartTime: str 173 | EndTime: str 174 | Features: str 175 | 176 | 177 | class JobSubmit(TypedDict): 178 | """A job submit record, from `compute/jobs`""" 179 | 180 | firecrest_taskid: str 181 | job_data_err: str 182 | job_data_out: str 183 | job_file: str 184 | job_file_err: str 185 | job_file_out: str 186 | jobid: int 187 | result: str 188 | 189 | 190 | class InternalTransferJobSubmit(JobSubmit): 191 | """A transfer job submit record, from `storage/xfer-internal/{op}`""" 192 | 193 | system: str 194 | 195 | 196 | class Id(TypedDict): 197 | name: str 198 | id: str 199 | 200 | 201 | class UserId(TypedDict): 202 | """A record from the `id` command""" 203 | 204 | user: Id 205 | group: Id 206 | groups: list[Id] 207 | -------------------------------------------------------------------------------- /firecrest/utilities.py: -------------------------------------------------------------------------------- 1 | import email.utils as eut 2 | import logging 3 | import time 4 | 5 | from contextlib import contextmanager 6 | from packaging.version import parse 7 | from xml.etree import ElementTree 8 | 9 | import firecrest.FirecrestException as fe 10 | 11 | 12 | @contextmanager 13 | def time_block(label, logger): 14 | start_time = time.time() 15 | try: 16 | yield 17 | finally: 18 | end_time = time.time() 19 | logger.debug(f"{label} took {end_time - start_time:.6f} seconds") 20 | 21 | 22 | def slurm_state_completed(state): 23 | completion_states = { 24 | 'BOOT_FAIL', 25 | 'CANCELLED', 26 | 'COMPLETED', 27 | 'DEADLINE', 28 | 'FAILED', 29 | 'NODE_FAIL', 30 | 'OUT_OF_MEMORY', 31 | 'PREEMPTED', 32 | 'TIMEOUT', 33 | } 34 | if state: 35 | # Make sure all the steps include one of the completion states 36 | return all( 37 | any(cs in s for cs in completion_states) for s in state.split(',') 38 | ) 39 | 40 | return False 41 | 42 | 43 | def parse_retry_after(retry_after_header, log_func): 44 | """ 45 | Parse the Retry-After header. 46 | 47 | :param retry_after_header: The value of the Retry-After header. 48 | :return: A non-negative floating point number representing when the retry 49 | should occur. 50 | """ 51 | try: 52 | # Try to parse it as a delta-seconds 53 | delta_seconds = int(retry_after_header) 54 | return max(delta_seconds, 0) 55 | except ValueError: 56 | pass 57 | 58 | try: 59 | # Try to parse it as an HTTP-date 60 | retry_after_date = eut.parsedate_to_datetime(retry_after_header) 61 | delta_seconds = retry_after_date.timestamp() - time.time() 62 | return max(delta_seconds, 0) 63 | except Exception: 64 | log_func( 65 | logging.WARNING, 66 | f"Could not parse Retry-After header: {retry_after_header}" 67 | ) 68 | return 10 69 | 70 | 71 | def validate_api_version_compatibility(): 72 | def decorator(func): 73 | def wrapper(*args, **kwargs): 74 | client = args[0] 75 | if client._query_api_version: 76 | # This will set the version in the client as a side 77 | # effect 78 | client.parameters() 79 | 80 | function_name = func.__name__ 81 | min_version = missing_api_features.get( 82 | function_name, {} 83 | ).get('min_version', None) 84 | 85 | if min_version and client._api_version < min_version: 86 | raise fe.NotImplementedOnAPIversion( 87 | f"function `{function_name}` is not available for " 88 | f"version <{min_version} in the client." 89 | ) 90 | 91 | return func(*args, **kwargs) 92 | return wrapper 93 | return decorator 94 | 95 | 96 | def async_validate_api_version_compatibility(): 97 | def decorator(func): 98 | async def wrapper(*args, **kwargs): 99 | 100 | client = args[0] 101 | if client._query_api_version: 102 | # This will set the version in the client as a side 103 | # effect 104 | await client.parameters() 105 | 106 | function_name = func.__name__ 107 | min_version = missing_api_features.get( 108 | function_name, {} 109 | ).get('min_version', None) 110 | 111 | if min_version and client._api_version < min_version: 112 | raise fe.NotImplementedOnAPIversion( 113 | f"function `{function_name}` is not available for " 114 | f"version <{min_version} in the client." 115 | ) 116 | 117 | return await func(*args, **kwargs) 118 | return wrapper 119 | return decorator 120 | 121 | 122 | missing_api_features = { 123 | 'compress': { 124 | # Using dictionaries in case we have max_version at 125 | # some point 126 | 'min_version': parse("1.16.0"), 127 | }, 128 | 'extract': { 129 | 'min_version': parse("1.16.0"), 130 | }, 131 | 'filesystems': { 132 | 'min_version': parse("1.15.0"), 133 | }, 134 | 'groups': { 135 | 'min_version': parse("1.15.0"), 136 | }, 137 | 'nodes': { 138 | 'min_version': parse("1.16.0"), 139 | }, 140 | 'partitions': { 141 | 'min_version': parse("1.16.0"), 142 | }, 143 | 'reservations': { 144 | 'min_version': parse("1.16.0"), 145 | }, 146 | 'submit_compress_job': { 147 | 'min_version': parse("1.16.0"), 148 | }, 149 | 'submit_extract_job': { 150 | 'min_version': parse("1.16.0"), 151 | }, 152 | } 153 | 154 | 155 | def part_checksum_xml(all_tags): 156 | root = ElementTree.Element( 157 | 'CompleteMultipartUpload', {'xmlns': "http://s3.amazonaws.com/doc/2006-03-01/"} 158 | ) 159 | for p in all_tags: 160 | part_element = ElementTree.SubElement(root, 'Part') 161 | part_etag = ElementTree.SubElement(part_element, 'ETag') 162 | part_etag.text = p['ETag'] 163 | part_number = ElementTree.SubElement(part_element, 'PartNumber') 164 | part_number.text = str(p['PartNumber']) 165 | return ElementTree.tostring( 166 | root, encoding='utf-8', xml_declaration=True, method='xml' 167 | ).decode('utf-8') 168 | -------------------------------------------------------------------------------- /firecrest/v1/AsyncExternalStorage.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019-2023, ETH Zurich. All rights reserved. 3 | # 4 | # Please, refer to the LICENSE file in the root directory. 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | # 7 | from __future__ import annotations 8 | 9 | import asyncio 10 | from io import BufferedWriter 11 | import logging 12 | import pathlib 13 | import requests 14 | import shutil 15 | import sys 16 | from typing import ContextManager, Optional, List, TYPE_CHECKING 17 | import urllib.request 18 | from packaging.version import Version 19 | 20 | if TYPE_CHECKING: 21 | from firecrest.v1.AsyncClient import AsyncFirecrest as AsyncFirecrestV1 22 | 23 | from contextlib import nullcontext 24 | from requests.compat import json # type: ignore 25 | 26 | if sys.version_info >= (3, 8): 27 | from typing import Literal 28 | else: 29 | from typing_extensions import Literal 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | class AsyncExternalStorage: 35 | """External storage object.""" 36 | 37 | _final_states: set[str] 38 | 39 | def __init__( 40 | self, 41 | client: AsyncFirecrestV1, 42 | task_id: str, 43 | previous_responses: Optional[List[requests.Response]] = None, 44 | ) -> None: 45 | previous_responses = [] if previous_responses is None else previous_responses 46 | self._client = client 47 | self._task_id = task_id 48 | self._in_progress = True 49 | self._status: Optional[str] = None 50 | self._data = None 51 | self._object_storage_data = None 52 | self._responses = previous_responses 53 | 54 | @property 55 | def client(self) -> AsyncFirecrestV1: 56 | """Returns the client that will be used to get information for the task.""" 57 | return self._client 58 | 59 | @property 60 | def task_id(self) -> str: 61 | """Returns the FirecREST task ID that is associated with this transfer.""" 62 | return self._task_id 63 | 64 | async def _update(self) -> None: 65 | if self._status not in self._final_states: 66 | task = await self._client._task_safe(self._task_id, self._responses) 67 | self._status = task["status"] 68 | self._data = task["data"] 69 | self._client.log( 70 | logging.INFO, 71 | f"Task {self._task_id} has status {self._status}" 72 | ) 73 | if not self._object_storage_data: 74 | if self._status == "111": 75 | self._object_storage_data = task["data"]["msg"] 76 | elif self._status == "117": 77 | self._object_storage_data = task["data"] 78 | 79 | @property 80 | async def status(self) -> str: 81 | """Returns status of the task that is associated with this transfer. 82 | 83 | :calls: GET `/tasks/{taskid}` 84 | """ 85 | await self._update() 86 | return self._status # type: ignore 87 | 88 | @property 89 | async def in_progress(self) -> bool: 90 | """Returns `False` when the transfer has been completed (succesfully or with errors), otherwise `True`. 91 | 92 | :calls: GET `/tasks/{taskid}` 93 | """ 94 | await self._update() 95 | return self._status not in self._final_states 96 | 97 | @property 98 | async def data(self) -> Optional[dict]: 99 | """Returns the task information from the latest response. 100 | 101 | :calls: GET `/tasks/{taskid}` 102 | """ 103 | await self._update() 104 | return self._data 105 | 106 | @property 107 | async def object_storage_data(self): 108 | """Returns the necessary information for the external transfer. 109 | The call is blocking and in cases of large file transfers it might take a long time. 110 | 111 | :calls: GET `/tasks/{taskid}` 112 | :rtype: dictionary or string 113 | """ 114 | if not self._object_storage_data: 115 | await self._update() 116 | 117 | while not self._object_storage_data: 118 | # No need for extra sleeping here, since the async client handles 119 | # the rate of requests anyway 120 | await self._update() 121 | 122 | return self._object_storage_data 123 | 124 | 125 | class AsyncExternalUpload(AsyncExternalStorage): 126 | """ 127 | This class handles the external upload from a file. 128 | 129 | Tracks the progress of the upload through the status of the associated task. 130 | Final states: *114* and *115*. 131 | 132 | +--------+--------------------------------------------------------------------+ 133 | | Status | Description | 134 | +========+====================================================================+ 135 | | 110 | Waiting for Form URL from Object Storage to be retrieved | 136 | +--------+--------------------------------------------------------------------+ 137 | | 111 | Form URL from Object Storage received | 138 | +--------+--------------------------------------------------------------------+ 139 | | 112 | Object Storage confirms that upload to Object Storage has finished | 140 | +--------+--------------------------------------------------------------------+ 141 | | 113 | Download from Object Storage to server has started | 142 | +--------+--------------------------------------------------------------------+ 143 | | 114 | Download from Object Storage to server has finished | 144 | +--------+--------------------------------------------------------------------+ 145 | | 115 | Download from Object Storage error | 146 | +--------+--------------------------------------------------------------------+ 147 | 148 | :param client: FirecREST client associated with the transfer 149 | :param task_id: FirecrREST task associated with the transfer 150 | """ 151 | 152 | def __init__( 153 | self, 154 | client: AsyncFirecrestV1, 155 | task_id: str, 156 | previous_responses: Optional[List[requests.Response]] = None, 157 | ) -> None: 158 | previous_responses = [] if previous_responses is None else previous_responses 159 | super().__init__(client, task_id, previous_responses) 160 | self._final_states = {"114", "115"} 161 | self._client.log( 162 | logging.INFO, 163 | f"Creating ExternalUpload object for task {task_id}" 164 | ) 165 | 166 | async def finish_upload(self) -> None: 167 | """Finish the upload process. 168 | This call will upload the file to the staging area. 169 | Check with the method `status` or `in_progress` to see the status of the transfer. 170 | The transfer from the staging area to the systems's filesystem can take several seconds to start to start. 171 | """ 172 | c = (await self.object_storage_data)["command"] # typer: ignore 173 | # LOCAL FIX FOR MAC 174 | # c = c.replace("192.168.220.19", "localhost") 175 | self._client.log( 176 | logging.INFO, 177 | f"Uploading the file to the staging area with the command: {c}" 178 | ) 179 | proc = await asyncio.create_subprocess_shell( 180 | c, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE 181 | ) 182 | 183 | _, stderr = await proc.communicate() 184 | if proc.returncode != 0: 185 | exc = Exception( 186 | f"Failed to finish upload with error: {stderr.decode('utf-8')}" 187 | ) 188 | self._client.log(logging.CRITICAL, exc) 189 | raise exc 190 | 191 | 192 | class AsyncExternalDownload(AsyncExternalStorage): 193 | """ 194 | This class handles the external download from a file. 195 | 196 | Tracks the progress of the download through the status of the associated task. 197 | Final states: *117* and *118*. 198 | 199 | +--------+--------------------------------------------------------------------+ 200 | | Status | Description | 201 | +========+====================================================================+ 202 | | 116 | Started upload from filesystem to Object Storage | 203 | +--------+--------------------------------------------------------------------+ 204 | | 117 | Upload from filesystem to Object Storage has finished successfully | 205 | +--------+--------------------------------------------------------------------+ 206 | | 118 | Upload from filesystem to Object Storage has finished with errors | 207 | +--------+--------------------------------------------------------------------+ 208 | 209 | :param client: FirecREST client associated with the transfer 210 | :param task_id: FirecrREST task associated with the transfer 211 | """ 212 | 213 | def __init__( 214 | self, 215 | client: AsyncFirecrestV1, 216 | task_id: str, 217 | previous_responses: Optional[List[requests.Response]] = None, 218 | ) -> None: 219 | previous_responses = [] if previous_responses is None else previous_responses 220 | super().__init__(client, task_id, previous_responses) 221 | self._final_states = {"117", "118"} 222 | self._client.log( 223 | logging.INFO, 224 | f"Creating ExternalDownload object for task {task_id}" 225 | ) 226 | 227 | async def invalidate_object_storage_link(self) -> None: 228 | """Invalidate the temporary URL for downloading. 229 | 230 | :calls: POST `/storage/xfer-external/invalidate` 231 | """ 232 | await self._client._invalidate(self._task_id) 233 | 234 | @property 235 | async def object_storage_link(self) -> str: 236 | """Get the direct download url for the file. The response from the FirecREST api 237 | changed after version 1.13.0, so make sure to set to older version, if you are 238 | using an older deployment. 239 | 240 | :calls: GET `/tasks/{taskid}` 241 | """ 242 | if self._client._api_version > Version("1.13.0"): 243 | return (await self.object_storage_data)["url"] 244 | else: 245 | return await self.object_storage_data 246 | 247 | async def finish_download( 248 | self, target_path: str | pathlib.Path | BufferedWriter 249 | ) -> None: 250 | """Finish the download process. 251 | 252 | :param target_path: the local path to save the file 253 | """ 254 | url = await self.object_storage_link 255 | self._client.log( 256 | logging.INFO, 257 | f"Downloading the file from {url} and saving to {target_path}" 258 | ) 259 | # LOCAL FIX FOR MAC 260 | # url = url.replace("192.168.220.19", "localhost") 261 | context: ContextManager[BufferedWriter] = ( 262 | open(target_path, "wb") # type: ignore 263 | if isinstance(target_path, str) or isinstance(target_path, pathlib.Path) 264 | else nullcontext(target_path) 265 | ) 266 | with urllib.request.urlopen(url) as response, context as out_file: 267 | shutil.copyfileobj(response, out_file) 268 | -------------------------------------------------------------------------------- /firecrest/v1/ExternalStorage.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019-2023, ETH Zurich. All rights reserved. 3 | # 4 | # Please, refer to the LICENSE file in the root directory. 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | # 7 | from __future__ import annotations 8 | 9 | from io import BufferedWriter 10 | import itertools 11 | import logging 12 | import pathlib 13 | import requests 14 | import shlex 15 | import shutil 16 | import subprocess 17 | import sys 18 | import time 19 | from typing import ContextManager, Optional, List, TYPE_CHECKING 20 | import urllib.request 21 | from packaging.version import Version 22 | 23 | if TYPE_CHECKING: 24 | from firecrest.v1.BasicClient import Firecrest as FirecrestV1 25 | 26 | from contextlib import nullcontext 27 | from requests.compat import json # type: ignore 28 | 29 | if sys.version_info >= (3, 8): 30 | from typing import Literal 31 | else: 32 | from typing_extensions import Literal 33 | 34 | logger = logging.getLogger(__name__) 35 | 36 | 37 | class ExternalStorage: 38 | """External storage object.""" 39 | 40 | _final_states: set[str] 41 | 42 | def __init__( 43 | self, 44 | client: FirecrestV1, 45 | task_id: str, 46 | previous_responses: Optional[List[requests.Response]] = None, 47 | ) -> None: 48 | previous_responses = [] if previous_responses is None else previous_responses 49 | self._client = client 50 | self._task_id = task_id 51 | self._in_progress = True 52 | self._status: Optional[str] = None 53 | self._data = None 54 | self._object_storage_data = None 55 | self._sleep_time = itertools.cycle([1]) 56 | self._responses = previous_responses 57 | 58 | @property 59 | def client(self) -> FirecrestV1: 60 | """Returns the client that will be used to get information for the task.""" 61 | return self._client 62 | 63 | @property 64 | def task_id(self) -> str: 65 | """Returns the FirecREST task ID that is associated with this transfer.""" 66 | return self._task_id 67 | 68 | def _update(self) -> None: 69 | if self._status not in self._final_states: 70 | task = self._client._task_safe(self._task_id, self._responses) 71 | self._status = task["status"] 72 | self._data = task["data"] 73 | self._client.log( 74 | logging.INFO, 75 | f"Task {self._task_id} has status {self._status}" 76 | ) 77 | if not self._object_storage_data: 78 | if self._status == "111": 79 | self._object_storage_data = task["data"]["msg"] 80 | elif self._status == "117": 81 | self._object_storage_data = task["data"] 82 | 83 | @property 84 | def status(self) -> str: 85 | """Returns status of the task that is associated with this transfer. 86 | 87 | :calls: GET `/tasks/{taskid}` 88 | """ 89 | self._update() 90 | return self._status # type: ignore 91 | 92 | @property 93 | def in_progress(self) -> bool: 94 | """Returns `False` when the transfer has been completed (succesfully or with errors), otherwise `True`. 95 | 96 | :calls: GET `/tasks/{taskid}` 97 | """ 98 | self._update() 99 | return self._status not in self._final_states 100 | 101 | @property 102 | def data(self) -> Optional[dict]: 103 | """Returns the task information from the latest response. 104 | 105 | :calls: GET `/tasks/{taskid}` 106 | """ 107 | self._update() 108 | return self._data 109 | 110 | @property 111 | def object_storage_data(self): 112 | """Returns the necessary information for the external transfer. 113 | The call is blocking and in cases of large file transfers it might take a long time. 114 | 115 | :calls: GET `/tasks/{taskid}` 116 | :rtype: dictionary or string 117 | """ 118 | if not self._object_storage_data: 119 | self._update() 120 | 121 | while not self._object_storage_data: 122 | t = next(self._sleep_time) 123 | self._client.log(logging.INFO, f"Sleeping for {t} sec") 124 | time.sleep(t) 125 | self._update() 126 | 127 | return self._object_storage_data 128 | 129 | 130 | class ExternalUpload(ExternalStorage): 131 | """ 132 | This class handles the external upload from a file. 133 | 134 | Tracks the progress of the upload through the status of the associated task. 135 | Final states: *114* and *115*. 136 | 137 | +--------+--------------------------------------------------------------------+ 138 | | Status | Description | 139 | +========+====================================================================+ 140 | | 110 | Waiting for Form URL from Object Storage to be retrieved | 141 | +--------+--------------------------------------------------------------------+ 142 | | 111 | Form URL from Object Storage received | 143 | +--------+--------------------------------------------------------------------+ 144 | | 112 | Object Storage confirms that upload to Object Storage has finished | 145 | +--------+--------------------------------------------------------------------+ 146 | | 113 | Download from Object Storage to server has started | 147 | +--------+--------------------------------------------------------------------+ 148 | | 114 | Download from Object Storage to server has finished | 149 | +--------+--------------------------------------------------------------------+ 150 | | 115 | Download from Object Storage error | 151 | +--------+--------------------------------------------------------------------+ 152 | 153 | :param client: FirecREST client associated with the transfer 154 | :param task_id: FirecrREST task associated with the transfer 155 | """ 156 | 157 | def __init__( 158 | self, 159 | client: FirecrestV1, 160 | task_id: str, 161 | previous_responses: Optional[List[requests.Response]] = None, 162 | ) -> None: 163 | previous_responses = [] if previous_responses is None else previous_responses 164 | super().__init__(client, task_id, previous_responses) 165 | self._final_states = {"114", "115"} 166 | self._client.log( 167 | logging.INFO, 168 | f"Creating ExternalUpload object for task {task_id}" 169 | ) 170 | 171 | def finish_upload(self) -> None: 172 | """Finish the upload process. 173 | This call will upload the file to the staging area. 174 | Check with the method `status` or `in_progress` to see the status of the transfer. 175 | The transfer from the staging area to the systems's filesystem can take several seconds to start to start. 176 | """ 177 | c = self.object_storage_data["command"] # typer: ignore 178 | # LOCAL FIX FOR MAC 179 | # c = c.replace("192.168.220.19", "localhost") 180 | self._client.log( 181 | logging.INFO, 182 | f"Uploading the file to the staging area with the command: {c}" 183 | ) 184 | command = subprocess.run( 185 | shlex.split(c), stdout=subprocess.PIPE, stderr=subprocess.PIPE 186 | ) 187 | if command.returncode != 0: 188 | exc = Exception( 189 | f"Failed to finish upload with error: {command.stderr.decode('utf-8')}" 190 | ) 191 | self._client.log(logging.CRITICAL, exc) 192 | raise exc 193 | 194 | 195 | class ExternalDownload(ExternalStorage): 196 | """ 197 | This class handles the external download from a file. 198 | 199 | Tracks the progress of the download through the status of the associated task. 200 | Final states: *117* and *118*. 201 | 202 | +--------+--------------------------------------------------------------------+ 203 | | Status | Description | 204 | +========+====================================================================+ 205 | | 116 | Started upload from filesystem to Object Storage | 206 | +--------+--------------------------------------------------------------------+ 207 | | 117 | Upload from filesystem to Object Storage has finished successfully | 208 | +--------+--------------------------------------------------------------------+ 209 | | 118 | Upload from filesystem to Object Storage has finished with errors | 210 | +--------+--------------------------------------------------------------------+ 211 | 212 | :param client: FirecREST client associated with the transfer 213 | :param task_id: FirecrREST task associated with the transfer 214 | """ 215 | 216 | def __init__( 217 | self, 218 | client: FirecrestV1, 219 | task_id: str, 220 | previous_responses: Optional[List[requests.Response]] = None, 221 | ) -> None: 222 | previous_responses = [] if previous_responses is None else previous_responses 223 | super().__init__(client, task_id, previous_responses) 224 | self._final_states = {"117", "118"} 225 | self._client.log( 226 | logging.INFO, 227 | f"Creating ExternalDownload object for task {task_id}" 228 | ) 229 | 230 | def invalidate_object_storage_link(self) -> None: 231 | """Invalidate the temporary URL for downloading. 232 | 233 | :calls: POST `/storage/xfer-external/invalidate` 234 | """ 235 | self._client._invalidate(self._task_id) 236 | 237 | @property 238 | def object_storage_link(self) -> str: 239 | """Get the direct download url for the file. The response from the FirecREST api 240 | changed after version 1.13.0, so make sure to set to older version, if you are 241 | using an older deployment. 242 | 243 | :calls: GET `/tasks/{taskid}` 244 | """ 245 | if self._client._api_version > Version("1.13.0"): 246 | return self.object_storage_data["url"] 247 | else: 248 | return self.object_storage_data 249 | 250 | def finish_download(self, target_path: str | pathlib.Path | BufferedWriter) -> None: 251 | """Finish the download process. The response from the FirecREST api changed after 252 | version 1.13.0, so make sure to set to older version, if you are using an older 253 | deployment. 254 | 255 | :param target_path: the local path to save the file 256 | 257 | :calls: GET `/tasks/{taskid}` 258 | """ 259 | url = self.object_storage_link 260 | self._client.log( 261 | logging.INFO, 262 | f"Downloading the file from {url} and saving to {target_path}" 263 | ) 264 | # LOCAL FIX FOR MAC 265 | # url = url.replace("192.168.220.19", "localhost") 266 | context: ContextManager[BufferedWriter] = ( 267 | open(target_path, "wb") # type: ignore 268 | if isinstance(target_path, str) or isinstance(target_path, pathlib.Path) 269 | else nullcontext(target_path) 270 | ) 271 | with urllib.request.urlopen(url) as response, context as out_file: 272 | shutil.copyfileobj(response, out_file) 273 | -------------------------------------------------------------------------------- /firecrest/v1/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2024, ETH Zurich. All rights reserved. 3 | # 4 | # Please, refer to the LICENSE file in the root directory. 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | # 7 | 8 | from firecrest.v1.AsyncClient import AsyncFirecrest 9 | from firecrest.v1.AsyncExternalStorage import ( 10 | AsyncExternalDownload, 11 | AsyncExternalUpload, 12 | AsyncExternalStorage, 13 | ) 14 | from firecrest.v1.BasicClient import Firecrest 15 | from firecrest.v1.ExternalStorage import ( 16 | ExternalDownload, 17 | ExternalUpload, 18 | ExternalStorage) 19 | 20 | -------------------------------------------------------------------------------- /firecrest/v2/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2024, ETH Zurich. All rights reserved. 3 | # 4 | # Please, refer to the LICENSE file in the root directory. 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | # 7 | 8 | from firecrest.v2._async.Client import ( # noqa 9 | AsyncExternalDownload, 10 | AsyncExternalUpload, 11 | AsyncFirecrest, 12 | ) 13 | from firecrest.v2._sync.Client import ( # noqa 14 | ExternalDownload, 15 | ExternalUpload, 16 | Firecrest, 17 | ) 18 | -------------------------------------------------------------------------------- /firecrest/v2/_async/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eth-cscs/pyfirecrest/961cf04913e987d5d54b9fcdd39c80a0cd7d382f/firecrest/v2/_async/__init__.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [tool.flit.module] 6 | name = "firecrest" 7 | 8 | [project] 9 | name = "pyfirecrest" 10 | dynamic = ["version"] 11 | description = "pyFirecrest is a python wrapper for FirecREST" 12 | authors = [{name = "CSCS Swiss National Supercomputing Center"}] 13 | maintainers = [ 14 | {name = "Eirini Koutsaniti", email = "eirini.koutsaniti@cscs.ch"}, 15 | {name = "Juan Pablo Dorsch", email = "juanpablo.dorsch@cscs.ch"} 16 | ] 17 | readme = "README.md" 18 | license = {file = "LICENSE"} 19 | classifiers = [ 20 | "Programming Language :: Python :: 3.7", 21 | "Programming Language :: Python :: 3.8", 22 | "Programming Language :: Python :: 3.9", 23 | "License :: OSI Approved :: BSD License", 24 | "Operating System :: OS Independent", 25 | ] 26 | requires-python = ">=3.7" 27 | dependencies = [ 28 | "aiofiles~=23.2.1", 29 | "requests>=2.14.0", 30 | "PyJWT>=2.4.0", 31 | "typer[all]~=0.7.0", 32 | "packaging>=21.0", 33 | "httpx>=0.24.0", 34 | "PyYAML>=5.1" 35 | ] 36 | 37 | [project.urls] 38 | Homepage = "https://pyfirecrest.readthedocs.io" 39 | Documentation = "https://pyfirecrest.readthedocs.io" 40 | Repository = "https://github.com/eth-cscs/pyfirecrest" 41 | 42 | [project.scripts] 43 | firecrest = "firecrest.cli_script:main" 44 | 45 | [project.optional-dependencies] 46 | test = [ 47 | "pytest>=5.3", 48 | "flake8~=5.0", 49 | "mypy~=0.991", 50 | "types-aiofiles~=23.2.0.0", 51 | "types-requests~=2.28.11", 52 | "pytest-httpserver~=1.0.6", 53 | "pytest-asyncio>=0.21.1", 54 | "types-PyYAML>=5.1" 55 | ] 56 | docs = [ 57 | "sphinx>=4.0", 58 | "sphinx-rtd-theme>=1.0", 59 | "myst-parser>=0.16", 60 | "sphinx-autobuild>=2021.0", 61 | "sphinx-click==3.0.2" 62 | ] 63 | dev = [ 64 | "unasync" 65 | ] 66 | 67 | [tool.mypy] 68 | show_error_codes = true 69 | strict = false 70 | exclude = [ 71 | "^docs/.*py$", 72 | "^tests/.*py$", 73 | ] 74 | 75 | [[tool.mypy.overrides]] 76 | module = [ 77 | "rich.*", 78 | ] 79 | ignore_missing_imports = true 80 | -------------------------------------------------------------------------------- /tests/common.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def clean_stdout(input): 5 | ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") 6 | return ansi_escape.sub("", input) 7 | -------------------------------------------------------------------------------- /tests/context.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) 5 | 6 | import firecrest 7 | -------------------------------------------------------------------------------- /tests/test_authorisation.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pytest 3 | 4 | from context import firecrest 5 | from werkzeug.wrappers import Response 6 | 7 | 8 | def auth_handler(request): 9 | client_id = request.form["client_id"] 10 | client_secret = request.form["client_secret"] 11 | if client_id == "valid_id": 12 | if client_secret == "valid_secret": 13 | ret = { 14 | "access_token": "VALID_TOKEN", 15 | "expires_in": 15, 16 | "refresh_expires_in": 0, 17 | "token_type": "Bearer", 18 | "not-before-policy": 0, 19 | "scope": "profile firecrest email", 20 | } 21 | ret_status = 200 22 | elif client_secret == "valid_secret_2": 23 | ret = { 24 | "access_token": "token_2", 25 | "expires_in": 15, 26 | "refresh_expires_in": 0, 27 | "token_type": "Bearer", 28 | "not-before-policy": 0, 29 | "scope": "profile firecrest email", 30 | } 31 | ret_status = 200 32 | else: 33 | ret = { 34 | "error": "unauthorized_client", 35 | "error_description": "Invalid client secret", 36 | } 37 | ret_status = 400 38 | else: 39 | ret = { 40 | "error": "invalid_client", 41 | "error_description": "Invalid client credentials", 42 | } 43 | ret_status = 400 44 | 45 | return Response(json.dumps(ret), status=ret_status, content_type="application/json") 46 | 47 | 48 | @pytest.fixture 49 | def auth_server(httpserver): 50 | httpserver.expect_request("/auth/token").respond_with_handler(auth_handler) 51 | return httpserver 52 | 53 | 54 | def test_client_credentials_valid(auth_server): 55 | auth_obj = firecrest.ClientCredentialsAuth( 56 | "valid_id", "valid_secret", auth_server.url_for("/auth/token") 57 | ) 58 | assert auth_obj._min_token_validity == 10 59 | assert auth_obj.get_access_token() == "VALID_TOKEN" 60 | # Change the secret differentiate between first and second request 61 | auth_obj._client_secret = "valid_secret_2" 62 | assert auth_obj.get_access_token() == "VALID_TOKEN" 63 | 64 | auth_obj = firecrest.ClientCredentialsAuth( 65 | "valid_id", 66 | "valid_secret", 67 | auth_server.url_for("/auth/token"), 68 | min_token_validity=20, 69 | ) 70 | assert auth_obj.get_access_token() == "VALID_TOKEN" 71 | # Change the secret differentiate between first and second request 72 | auth_obj._client_secret = "valid_secret_2" 73 | assert auth_obj.get_access_token() == "token_2" 74 | 75 | 76 | def test_client_credentials_invalid_id(auth_server): 77 | auth_obj = firecrest.ClientCredentialsAuth( 78 | "invalid_id", "valid_secret", auth_server.url_for("/auth/token") 79 | ) 80 | with pytest.raises(Exception) as exc_info: 81 | auth_obj.get_access_token() 82 | 83 | assert "Client credentials error" in str(exc_info.value) 84 | 85 | 86 | def test_client_credentials_invalid_secret(auth_server): 87 | auth_obj = firecrest.ClientCredentialsAuth( 88 | "valid_id", "invalid_secret", auth_server.url_for("/auth/token") 89 | ) 90 | with pytest.raises(Exception) as exc_info: 91 | auth_obj.get_access_token() 92 | 93 | assert "Client credentials error" in str(exc_info.value) 94 | -------------------------------------------------------------------------------- /tests/test_extras.py: -------------------------------------------------------------------------------- 1 | import common 2 | import json 3 | import pytest 4 | import re 5 | import test_authorisation as auth 6 | 7 | from context import firecrest 8 | from typer.testing import CliRunner 9 | from werkzeug.wrappers import Response 10 | from werkzeug.wrappers import Request 11 | 12 | from firecrest import __app_name__, __version__, cli 13 | 14 | 15 | runner = CliRunner() 16 | 17 | 18 | def test_cli_version(): 19 | result = runner.invoke(cli.app, ["--version"]) 20 | assert result.exit_code == 0 21 | assert f"FirecREST CLI Version: {__version__}\n" in result.stdout 22 | 23 | 24 | @pytest.fixture 25 | def client1(fc_server): 26 | class ValidAuthorization: 27 | def get_access_token(self): 28 | # This token was created in https://jwt.io/ with payload: 29 | # { 30 | # "realm_access": { 31 | # "roles": [ 32 | # "firecrest-sa" 33 | # ] 34 | # }, 35 | # "resource_access": { 36 | # "bob-client": { 37 | # "roles": [ 38 | # "bob" 39 | # ] 40 | # } 41 | # }, 42 | # "clientId": "bob-client", 43 | # "preferred_username": "service-account-bob-client" 44 | # } 45 | return "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiZmlyZWNyZXN0LXNhIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYm9iLWNsaWVudCI6eyJyb2xlcyI6WyJib2IiXX19LCJjbGllbnRJZCI6ImJvYi1jbGllbnQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJzZXJ2aWNlLWFjY291bnQtYm9iLWNsaWVudCJ9.XfCXDclEBh7faQrOF2piYdnb7c3AUiCxDesTkNSwpSY" 46 | 47 | return firecrest.v1.Firecrest( 48 | firecrest_url=fc_server.url_for("/"), authorization=ValidAuthorization() 49 | ) 50 | 51 | 52 | @pytest.fixture 53 | def client2(fc_server): 54 | class ValidAuthorization: 55 | def get_access_token(self): 56 | # This token was created in https://jwt.io/ with payload: 57 | # { 58 | # "realm_access": { 59 | # "roles": [ 60 | # "other-role" 61 | # ] 62 | # }, 63 | # "preferred_username": "alice" 64 | # } 65 | return "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib3RoZXItcm9sZSJdfSwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxpY2UifQ.dpo1_F9jkV-RpNGqTaCNLbM-JPMnstDg7mQjzbwDp5g" 66 | 67 | return firecrest.v1.Firecrest( 68 | firecrest_url=fc_server.url_for("/"), authorization=ValidAuthorization() 69 | ) 70 | 71 | 72 | @pytest.fixture 73 | def client3(fc_server): 74 | class ValidAuthorization: 75 | def get_access_token(self): 76 | # This token was created in https://jwt.io/ with payload: 77 | # { 78 | # "preferred_username": "eve" 79 | # } 80 | return "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwcmVmZXJyZWRfdXNlcm5hbWUiOiJldmUifQ.SGVPDrJdy8b5jRpxcw9ILLsf8M2ljAYWxiN0A1b_1SE" 81 | 82 | return firecrest.v1.Firecrest( 83 | firecrest_url=fc_server.url_for("/"), authorization=ValidAuthorization() 84 | ) 85 | 86 | 87 | @pytest.fixture 88 | def valid_client(fc_server): 89 | class ValidAuthorization: 90 | def get_access_token(self): 91 | return "VALID_TOKEN" 92 | 93 | client = firecrest.v1.Firecrest( 94 | firecrest_url=fc_server.url_for("/"), authorization=ValidAuthorization() 95 | ) 96 | client.set_api_version("1.16.0") 97 | return client 98 | 99 | 100 | @pytest.fixture 101 | def valid_credentials(fc_server, auth_server): 102 | return [ 103 | f"--firecrest-url={fc_server.url_for('/')}", 104 | "--client-id=valid_id", 105 | "--client-secret=valid_secret", 106 | f"--token-url={auth_server.url_for('/auth/token')}", 107 | "--api-version=1.16.0", 108 | ] 109 | 110 | 111 | @pytest.fixture 112 | def invalid_client(fc_server): 113 | class InvalidAuthorization: 114 | def get_access_token(self): 115 | return "INVALID_TOKEN" 116 | 117 | client = firecrest.v1.Firecrest( 118 | firecrest_url=fc_server.url_for("/"), authorization=InvalidAuthorization() 119 | ) 120 | client.set_api_version("1.16.0") 121 | return client 122 | 123 | 124 | def tasks_handler(request: Request): 125 | if request.headers["Authorization"] != "Bearer VALID_TOKEN": 126 | return Response( 127 | json.dumps({"message": "Bad token; invalid JSON"}), 128 | status=401, 129 | content_type="application/json", 130 | ) 131 | 132 | all_tasks = { 133 | "taskid_1": { 134 | "created_at": "2022-08-16T07:18:54", 135 | "data": "data", 136 | "description": "description", 137 | "hash_id": "taskid_1", 138 | "last_modify": "2022-08-16T07:18:54", 139 | "service": "storage", 140 | "status": "114", 141 | "system": "cluster1", 142 | "task_id": "taskid_1", 143 | "task_url": "TASK_IP/tasks/taskid_1", 144 | "updated_at": "2022-08-16T07:18:54", 145 | "user": "username", 146 | }, 147 | "taskid_2": { 148 | "created_at": "2022-08-16T07:18:54", 149 | "data": "data", 150 | "description": "description", 151 | "hash_id": "taskid_2", 152 | "last_modify": "2022-08-16T07:18:54", 153 | "service": "storage", 154 | "status": "112", 155 | "system": "cluster1", 156 | "task_id": "taskid_2", 157 | "task_url": "TASK_IP/tasks/taskid_2", 158 | "updated_at": "2022-08-16T07:18:54", 159 | "user": "username", 160 | }, 161 | "taskid_3": { 162 | "created_at": "2022-08-16T07:18:54", 163 | "data": "data", 164 | "description": "description", 165 | "hash_id": "taskid_3", 166 | "last_modify": "2022-08-16T07:18:54", 167 | "service": "storage", 168 | "status": "111", 169 | "system": "cluster1", 170 | "task_id": "taskid_3", 171 | "task_url": "TASK_IP/tasks/taskid_3", 172 | "updated_at": "2022-08-16T07:18:54", 173 | "user": "username", 174 | } 175 | } 176 | status_code = 200 177 | tasks = request.args.get("tasks") 178 | 179 | if tasks: 180 | tasks = tasks.split(",") 181 | ret = { 182 | "tasks": {k: v for k, v in all_tasks.items() if k in tasks} 183 | } 184 | else: 185 | ret = { 186 | "tasks": all_tasks 187 | } 188 | 189 | return Response( 190 | json.dumps(ret), status=status_code, content_type="application/json" 191 | ) 192 | 193 | 194 | @pytest.fixture 195 | def fc_server(httpserver): 196 | httpserver.expect_request( 197 | re.compile("^/tasks"), method="GET" 198 | ).respond_with_handler(tasks_handler) 199 | 200 | return httpserver 201 | 202 | 203 | @pytest.fixture 204 | def auth_server(httpserver): 205 | httpserver.expect_request("/auth/token").respond_with_handler(auth.auth_handler) 206 | return httpserver 207 | 208 | 209 | def test_whoami(client1): 210 | assert client1.whoami() == "bob" 211 | 212 | 213 | def test_whoami_2(client2): 214 | assert client2.whoami() == "alice" 215 | 216 | 217 | def test_whoami_3(client3): 218 | assert client3.whoami() == "eve" 219 | 220 | 221 | def test_whoami_invalid_client(invalid_client): 222 | assert invalid_client.whoami() == None 223 | 224 | 225 | def test_all_tasks(valid_client): 226 | assert valid_client._tasks() == { 227 | "taskid_1": { 228 | "created_at": "2022-08-16T07:18:54", 229 | "data": "data", 230 | "description": "description", 231 | "hash_id": "taskid_1", 232 | "last_modify": "2022-08-16T07:18:54", 233 | "service": "storage", 234 | "status": "114", 235 | "system": "cluster1", 236 | "task_id": "taskid_1", 237 | "task_url": "TASK_IP/tasks/taskid_1", 238 | "updated_at": "2022-08-16T07:18:54", 239 | "user": "username", 240 | }, 241 | "taskid_2": { 242 | "created_at": "2022-08-16T07:18:54", 243 | "data": "data", 244 | "description": "description", 245 | "hash_id": "taskid_2", 246 | "last_modify": "2022-08-16T07:18:54", 247 | "service": "storage", 248 | "status": "112", 249 | "system": "cluster1", 250 | "task_id": "taskid_2", 251 | "task_url": "TASK_IP/tasks/taskid_2", 252 | "updated_at": "2022-08-16T07:18:54", 253 | "user": "username", 254 | }, 255 | "taskid_3": { 256 | "created_at": "2022-08-16T07:18:54", 257 | "data": "data", 258 | "description": "description", 259 | "hash_id": "taskid_3", 260 | "last_modify": "2022-08-16T07:18:54", 261 | "service": "storage", 262 | "status": "111", 263 | "system": "cluster1", 264 | "task_id": "taskid_3", 265 | "task_url": "TASK_IP/tasks/taskid_3", 266 | "updated_at": "2022-08-16T07:18:54", 267 | "user": "username", 268 | }, 269 | } 270 | 271 | 272 | def test_cli_all_tasks(valid_credentials): 273 | args = valid_credentials + ["tasks", "--no-pager"] 274 | result = runner.invoke(cli.app, args=args) 275 | stdout = common.clean_stdout(result.stdout) 276 | assert result.exit_code == 0 277 | assert "Task information: 3 results" in stdout 278 | assert "Task ID | Status" in stdout 279 | assert "taskid_1 | 114" in stdout 280 | assert "taskid_2 | 112" in stdout 281 | assert "taskid_3 | 111" in stdout 282 | 283 | 284 | def test_subset_tasks(valid_client): 285 | # "taskid_4" is not a valid id but it will be silently ignored 286 | assert valid_client._tasks(["taskid_1", "taskid_3", "taskid_4"]) == { 287 | "taskid_1": { 288 | "created_at": "2022-08-16T07:18:54", 289 | "data": "data", 290 | "description": "description", 291 | "hash_id": "taskid_1", 292 | "last_modify": "2022-08-16T07:18:54", 293 | "service": "storage", 294 | "status": "114", 295 | "system": "cluster1", 296 | "task_id": "taskid_1", 297 | "task_url": "TASK_IP/tasks/taskid_1", 298 | "updated_at": "2022-08-16T07:18:54", 299 | "user": "username", 300 | }, 301 | "taskid_3": { 302 | "created_at": "2022-08-16T07:18:54", 303 | "data": "data", 304 | "description": "description", 305 | "hash_id": "taskid_3", 306 | "last_modify": "2022-08-16T07:18:54", 307 | "service": "storage", 308 | "status": "111", 309 | "system": "cluster1", 310 | "task_id": "taskid_3", 311 | "task_url": "TASK_IP/tasks/taskid_3", 312 | "updated_at": "2022-08-16T07:18:54", 313 | "user": "username", 314 | }, 315 | } 316 | 317 | 318 | def test_cli_subset_tasks(valid_credentials): 319 | args = valid_credentials + ["tasks", "--no-pager", "taskid_1", "taskid_3"] 320 | result = runner.invoke(cli.app, args=args) 321 | stdout = common.clean_stdout(result.stdout) 322 | assert result.exit_code == 0 323 | assert "Task information: 2 results" in stdout 324 | assert "Task ID | Status" in stdout 325 | assert "taskid_1 | 114" in stdout 326 | assert "taskid_3 | 111" in stdout 327 | assert "taskid_2 | 112" not in stdout 328 | 329 | 330 | def test_one_task(valid_client): 331 | assert valid_client._tasks(["taskid_2"]) == { 332 | "taskid_2": { 333 | "created_at": "2022-08-16T07:18:54", 334 | "data": "data", 335 | "description": "description", 336 | "hash_id": "taskid_2", 337 | "last_modify": "2022-08-16T07:18:54", 338 | "service": "storage", 339 | "status": "112", 340 | "system": "cluster1", 341 | "task_id": "taskid_2", 342 | "task_url": "TASK_IP/tasks/taskid_2", 343 | "updated_at": "2022-08-16T07:18:54", 344 | "user": "username", 345 | } 346 | } 347 | 348 | 349 | def test_cli_one_task(valid_credentials): 350 | args = valid_credentials + ["tasks", "--no-pager", "taskid_2"] 351 | result = runner.invoke(cli.app, args=args) 352 | stdout = common.clean_stdout(result.stdout) 353 | assert result.exit_code == 0 354 | assert "Task information: 1 result" in stdout 355 | assert "Task ID | Status" in stdout 356 | assert "taskid_2 | 112" in stdout 357 | assert "taskid_1 | 114" not in stdout 358 | assert "taskid_3 | 111" not in stdout 359 | 360 | 361 | def test_invalid_task(valid_client): 362 | assert valid_client._tasks(["invalid_id"]) == {} 363 | 364 | 365 | def test_tasks_invalid(invalid_client): 366 | with pytest.raises(firecrest.UnauthorizedException): 367 | invalid_client._tasks() 368 | -------------------------------------------------------------------------------- /tests/test_extras_async.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import re 3 | import test_extras as basic_extras 4 | 5 | from context import firecrest 6 | 7 | from firecrest import __app_name__, __version__ 8 | 9 | 10 | @pytest.fixture 11 | def valid_client(fc_server): 12 | class ValidAuthorization: 13 | def get_access_token(self): 14 | return "VALID_TOKEN" 15 | 16 | client = firecrest.v1.AsyncFirecrest( 17 | firecrest_url=fc_server.url_for("/"), authorization=ValidAuthorization() 18 | ) 19 | client.time_between_calls = { 20 | "compute": 0, 21 | "reservations": 0, 22 | "status": 0, 23 | "storage": 0, 24 | "tasks": 0, 25 | "utilities": 0, 26 | } 27 | client.set_api_version("1.16.0") 28 | 29 | return client 30 | 31 | 32 | @pytest.fixture 33 | def invalid_client(fc_server): 34 | class InvalidAuthorization: 35 | def get_access_token(self): 36 | return "INVALID_TOKEN" 37 | 38 | client = firecrest.v1.AsyncFirecrest( 39 | firecrest_url=fc_server.url_for("/"), authorization=InvalidAuthorization() 40 | ) 41 | client.time_between_calls = { 42 | "compute": 0, 43 | "reservations": 0, 44 | "status": 0, 45 | "storage": 0, 46 | "tasks": 0, 47 | "utilities": 0, 48 | } 49 | client.set_api_version("1.16.0") 50 | 51 | return client 52 | 53 | 54 | @pytest.fixture 55 | def fc_server(httpserver): 56 | httpserver.expect_request( 57 | "/tasks", method="GET" 58 | ).respond_with_handler(basic_extras.tasks_handler) 59 | 60 | return httpserver 61 | 62 | 63 | @pytest.mark.asyncio 64 | async def test_all_tasks(valid_client): 65 | assert await valid_client._tasks() == { 66 | "taskid_1": { 67 | "created_at": "2022-08-16T07:18:54", 68 | "data": "data", 69 | "description": "description", 70 | "hash_id": "taskid_1", 71 | "last_modify": "2022-08-16T07:18:54", 72 | "service": "storage", 73 | "status": "114", 74 | "system": "cluster1", 75 | "task_id": "taskid_1", 76 | "task_url": "TASK_IP/tasks/taskid_1", 77 | "updated_at": "2022-08-16T07:18:54", 78 | "user": "username", 79 | }, 80 | "taskid_2": { 81 | "created_at": "2022-08-16T07:18:54", 82 | "data": "data", 83 | "description": "description", 84 | "hash_id": "taskid_2", 85 | "last_modify": "2022-08-16T07:18:54", 86 | "service": "storage", 87 | "status": "112", 88 | "system": "cluster1", 89 | "task_id": "taskid_2", 90 | "task_url": "TASK_IP/tasks/taskid_2", 91 | "updated_at": "2022-08-16T07:18:54", 92 | "user": "username", 93 | }, 94 | "taskid_3": { 95 | "created_at": "2022-08-16T07:18:54", 96 | "data": "data", 97 | "description": "description", 98 | "hash_id": "taskid_3", 99 | "last_modify": "2022-08-16T07:18:54", 100 | "service": "storage", 101 | "status": "111", 102 | "system": "cluster1", 103 | "task_id": "taskid_3", 104 | "task_url": "TASK_IP/tasks/taskid_3", 105 | "updated_at": "2022-08-16T07:18:54", 106 | "user": "username", 107 | }, 108 | } 109 | 110 | 111 | @pytest.mark.asyncio 112 | async def test_subset_tasks(valid_client): 113 | # "taskid_4" is not a valid id but it will be silently ignored 114 | assert await valid_client._tasks(["taskid_1", "taskid_3", "taskid_4"]) == { 115 | "taskid_1": { 116 | "created_at": "2022-08-16T07:18:54", 117 | "data": "data", 118 | "description": "description", 119 | "hash_id": "taskid_1", 120 | "last_modify": "2022-08-16T07:18:54", 121 | "service": "storage", 122 | "status": "114", 123 | "system": "cluster1", 124 | "task_id": "taskid_1", 125 | "task_url": "TASK_IP/tasks/taskid_1", 126 | "updated_at": "2022-08-16T07:18:54", 127 | "user": "username", 128 | }, 129 | "taskid_3": { 130 | "created_at": "2022-08-16T07:18:54", 131 | "data": "data", 132 | "description": "description", 133 | "hash_id": "taskid_3", 134 | "last_modify": "2022-08-16T07:18:54", 135 | "service": "storage", 136 | "status": "111", 137 | "system": "cluster1", 138 | "task_id": "taskid_3", 139 | "task_url": "TASK_IP/tasks/taskid_3", 140 | "updated_at": "2022-08-16T07:18:54", 141 | "user": "username", 142 | }, 143 | } 144 | 145 | 146 | @pytest.mark.asyncio 147 | async def test_one_task(valid_client): 148 | assert await valid_client._tasks(["taskid_2"]) == { 149 | "taskid_2": { 150 | "created_at": "2022-08-16T07:18:54", 151 | "data": "data", 152 | "description": "description", 153 | "hash_id": "taskid_2", 154 | "last_modify": "2022-08-16T07:18:54", 155 | "service": "storage", 156 | "status": "112", 157 | "system": "cluster1", 158 | "task_id": "taskid_2", 159 | "task_url": "TASK_IP/tasks/taskid_2", 160 | "updated_at": "2022-08-16T07:18:54", 161 | "user": "username", 162 | } 163 | } 164 | 165 | 166 | @pytest.mark.asyncio 167 | async def test_invalid_task(valid_client): 168 | assert await valid_client._tasks(["invalid_id"]) == {} 169 | 170 | 171 | @pytest.mark.asyncio 172 | async def test_tasks_invalid(invalid_client): 173 | with pytest.raises(firecrest.UnauthorizedException): 174 | await invalid_client._tasks() 175 | -------------------------------------------------------------------------------- /tests/test_status_async.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import re 3 | import test_status as basic_status 4 | 5 | from context import firecrest 6 | from firecrest import __app_name__, __version__ 7 | 8 | 9 | @pytest.fixture 10 | def valid_client(fc_server): 11 | class ValidAuthorization: 12 | def get_access_token(self): 13 | return "VALID_TOKEN" 14 | 15 | client = firecrest.v1.AsyncFirecrest( 16 | firecrest_url=fc_server.url_for("/"), authorization=ValidAuthorization() 17 | ) 18 | client.time_between_calls = { 19 | "compute": 0, 20 | "reservations": 0, 21 | "status": 0, 22 | "storage": 0, 23 | "tasks": 0, 24 | "utilities": 0, 25 | } 26 | client.set_api_version("1.16.0") 27 | 28 | return client 29 | 30 | 31 | @pytest.fixture 32 | def invalid_client(fc_server): 33 | class InvalidAuthorization: 34 | def get_access_token(self): 35 | return "INVALID_TOKEN" 36 | 37 | client = firecrest.v1.AsyncFirecrest( 38 | firecrest_url=fc_server.url_for("/"), authorization=InvalidAuthorization() 39 | ) 40 | client.time_between_calls = { 41 | "compute": 0, 42 | "reservations": 0, 43 | "status": 0, 44 | "storage": 0, 45 | "tasks": 0, 46 | "utilities": 0, 47 | } 48 | client.set_api_version("1.16.0") 49 | 50 | return client 51 | 52 | 53 | @pytest.fixture 54 | def fc_server(httpserver): 55 | httpserver.expect_request( 56 | re.compile("^/status/services.*"), method="GET" 57 | ).respond_with_handler(basic_status.services_handler) 58 | 59 | httpserver.expect_request( 60 | re.compile("^/status/systems.*"), method="GET" 61 | ).respond_with_handler(basic_status.systems_handler) 62 | 63 | httpserver.expect_request("/status/parameters", method="GET").respond_with_handler( 64 | basic_status.parameters_handler 65 | ) 66 | 67 | httpserver.expect_request( 68 | re.compile("^/status/filesystems.*"), method="GET" 69 | ).respond_with_handler(basic_status.filesystems_handler) 70 | 71 | return httpserver 72 | 73 | 74 | @pytest.mark.asyncio 75 | async def test_all_services(valid_client): 76 | assert await valid_client.all_services() == [ 77 | { 78 | "description": "server up & flask running", 79 | "service": "utilities", 80 | "status": "available", 81 | }, 82 | { 83 | "description": "server up & flask running", 84 | "service": "compute", 85 | "status": "available", 86 | }, 87 | ] 88 | 89 | 90 | @pytest.mark.asyncio 91 | async def test_all_services_invalid(invalid_client): 92 | with pytest.raises(firecrest.UnauthorizedException): 93 | await invalid_client.all_services() 94 | 95 | 96 | @pytest.mark.asyncio 97 | async def test_service(valid_client): 98 | assert await valid_client.service("utilities") == { 99 | "description": "server up & flask running", 100 | "service": "utilities", 101 | "status": "available", 102 | } 103 | 104 | 105 | @pytest.mark.asyncio 106 | async def test_invalid_service(valid_client): 107 | with pytest.raises(firecrest.FirecrestException): 108 | await valid_client.service("invalid_service") 109 | 110 | 111 | @pytest.mark.asyncio 112 | async def test_service_invalid(invalid_client): 113 | with pytest.raises(firecrest.UnauthorizedException): 114 | await invalid_client.service("utilities") 115 | 116 | 117 | @pytest.mark.asyncio 118 | async def test_all_systems(valid_client): 119 | assert await valid_client.all_systems() == [ 120 | {"description": "System ready", "status": "available", "system": "cluster1"}, 121 | {"description": "System ready", "status": "available", "system": "cluster2"}, 122 | ] 123 | 124 | 125 | @pytest.mark.asyncio 126 | async def test_all_systems_invalid(invalid_client): 127 | with pytest.raises(firecrest.UnauthorizedException): 128 | await invalid_client.all_systems() 129 | 130 | 131 | @pytest.mark.asyncio 132 | async def test_system(valid_client): 133 | assert await valid_client.system("cluster1") == { 134 | "description": "System ready", 135 | "status": "available", 136 | "system": "cluster1", 137 | } 138 | 139 | 140 | @pytest.mark.asyncio 141 | async def test_invalid_system(valid_client): 142 | with pytest.raises(firecrest.FirecrestException): 143 | await valid_client.system("invalid_system") 144 | 145 | 146 | @pytest.mark.asyncio 147 | async def test_system_invalid(invalid_client): 148 | with pytest.raises(firecrest.UnauthorizedException): 149 | await invalid_client.system("cluster1") 150 | 151 | 152 | @pytest.mark.asyncio 153 | async def test_parameters(valid_client): 154 | assert await valid_client.parameters() == { 155 | "storage": [ 156 | {"name": "OBJECT_STORAGE", "unit": "", "value": "swift"}, 157 | {"name": "STORAGE_TEMPURL_EXP_TIME", "unit": "seconds", "value": "2592000"}, 158 | {"name": "STORAGE_MAX_FILE_SIZE", "unit": "MB", "value": "512000"}, 159 | { 160 | "name": "FILESYSTEMS", 161 | "unit": "", 162 | "value": [{"mounted": ["/fs1"], "system": "cluster1"}], 163 | }, 164 | ], 165 | "utilities": [ 166 | {"name": "UTILITIES_MAX_FILE_SIZE", "unit": "MB", "value": "5"}, 167 | {"name": "UTILITIES_TIMEOUT", "unit": "seconds", "value": "5"}, 168 | ], 169 | } 170 | 171 | 172 | @pytest.mark.asyncio 173 | async def test_parameters_invalid(invalid_client): 174 | with pytest.raises(firecrest.UnauthorizedException): 175 | await invalid_client.parameters() 176 | 177 | 178 | @pytest.mark.asyncio 179 | async def test_filesystems(valid_client): 180 | assert await valid_client.filesystems() == { 181 | "cluster": [ 182 | { 183 | "description": "Users home filesystem", 184 | "name": "HOME", 185 | "path": "/home", 186 | "status": "available", 187 | "status_code": 200 188 | }, 189 | { 190 | "description": "Scratch filesystem", 191 | "name": "SCRATCH", 192 | "path": "/scratch", 193 | "status": "not available", 194 | "status_code": 400 195 | } 196 | ] 197 | } 198 | 199 | assert await valid_client.filesystems(system_name="cluster") == { 200 | "cluster": [ 201 | { 202 | "description": "Users home filesystem", 203 | "name": "HOME", 204 | "path": "/home", 205 | "status": "available", 206 | "status_code": 200 207 | }, 208 | { 209 | "description": "Scratch filesystem", 210 | "name": "SCRATCH", 211 | "path": "/scratch", 212 | "status": "not available", 213 | "status_code": 400 214 | } 215 | ] 216 | } 217 | 218 | 219 | @pytest.mark.asyncio 220 | async def test_filesystems_invalid(invalid_client): 221 | with pytest.raises(firecrest.UnauthorizedException): 222 | await invalid_client.filesystems() 223 | -------------------------------------------------------------------------------- /tests/test_storage_async.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import re 3 | import test_storage as basic_storage 4 | 5 | from context import firecrest 6 | 7 | from firecrest import __app_name__, __version__ 8 | 9 | 10 | @pytest.fixture 11 | def valid_client(fc_server): 12 | class ValidAuthorization: 13 | def get_access_token(self): 14 | return "VALID_TOKEN" 15 | 16 | client = firecrest.v1.AsyncFirecrest( 17 | firecrest_url=fc_server.url_for("/"), authorization=ValidAuthorization() 18 | ) 19 | client.time_between_calls = { 20 | "compute": 0, 21 | "reservations": 0, 22 | "status": 0, 23 | "storage": 0, 24 | "tasks": 0, 25 | "utilities": 0, 26 | } 27 | client.set_api_version("1.16.0") 28 | 29 | return client 30 | 31 | 32 | @pytest.fixture 33 | def invalid_client(fc_server): 34 | class InvalidAuthorization: 35 | def get_access_token(self): 36 | return "INVALID_TOKEN" 37 | 38 | client = firecrest.v1.AsyncFirecrest( 39 | firecrest_url=fc_server.url_for("/"), authorization=InvalidAuthorization() 40 | ) 41 | client.time_between_calls = { 42 | "compute": 0, 43 | "reservations": 0, 44 | "status": 0, 45 | "storage": 0, 46 | "tasks": 0, 47 | "utilities": 0, 48 | } 49 | client.set_api_version("1.16.0") 50 | 51 | return client 52 | 53 | 54 | @pytest.fixture 55 | def fc_server(httpserver): 56 | httpserver.expect_request( 57 | re.compile("^/storage/xfer-internal.*"), method="POST" 58 | ).respond_with_handler(basic_storage.internal_transfer_handler) 59 | 60 | httpserver.expect_request( 61 | "/tasks", method="GET" 62 | ).respond_with_handler(basic_storage.storage_tasks_handler) 63 | 64 | httpserver.expect_request( 65 | "/storage/xfer-external/download", method="POST" 66 | ).respond_with_handler(basic_storage.external_download_handler) 67 | 68 | httpserver.expect_request( 69 | "/storage/xfer-external/upload", method="POST" 70 | ).respond_with_handler(basic_storage.external_upload_handler) 71 | 72 | return httpserver 73 | 74 | 75 | @pytest.mark.asyncio 76 | async def test_internal_transfer(valid_client): 77 | global internal_transfer_retry 78 | 79 | # mv job 80 | internal_transfer_retry = 0 81 | assert await valid_client.submit_move_job( 82 | machine="cluster1", 83 | source_path="/path/to/source", 84 | target_path="/path/to/destination", 85 | job_name="mv-job", 86 | time="2", 87 | stage_out_job_id="35363851", 88 | account="project", 89 | ) == { 90 | "job_data_err": "", 91 | "job_data_out": "", 92 | "job_file": "/path/to/firecrest/internal_transfer_id/sbatch-job.sh", 93 | "job_file_err": "/path/to/firecrest/internal_transfer_id/job-35363861.err", 94 | "job_file_out": "/path/to/firecrest/internal_transfer_id/job-35363861.out", 95 | "jobid": 35363861, 96 | "result": "Job submitted", 97 | "system": "cluster1", 98 | } 99 | 100 | # cp job 101 | internal_transfer_retry = 0 102 | assert await valid_client.submit_copy_job( 103 | machine="cluster1", 104 | source_path="/path/to/source", 105 | target_path="/path/to/destination", 106 | job_name="mv-job", 107 | time="2", 108 | stage_out_job_id="35363851", 109 | account="project", 110 | ) == { 111 | "job_data_err": "", 112 | "job_data_out": "", 113 | "job_file": "/path/to/firecrest/internal_transfer_id/sbatch-job.sh", 114 | "job_file_err": "/path/to/firecrest/internal_transfer_id/job-35363861.err", 115 | "job_file_out": "/path/to/firecrest/internal_transfer_id/job-35363861.out", 116 | "jobid": 35363861, 117 | "result": "Job submitted", 118 | "system": "cluster1", 119 | } 120 | 121 | # rsync job 122 | internal_transfer_retry = 0 123 | assert await valid_client.submit_rsync_job( 124 | machine="cluster1", 125 | source_path="/path/to/source", 126 | target_path="/path/to/destination", 127 | job_name="mv-job", 128 | time="2", 129 | stage_out_job_id="35363851", 130 | account="project", 131 | ) == { 132 | "job_data_err": "", 133 | "job_data_out": "", 134 | "job_file": "/path/to/firecrest/internal_transfer_id/sbatch-job.sh", 135 | "job_file_err": "/path/to/firecrest/internal_transfer_id/job-35363861.err", 136 | "job_file_out": "/path/to/firecrest/internal_transfer_id/job-35363861.out", 137 | "jobid": 35363861, 138 | "result": "Job submitted", 139 | "system": "cluster1", 140 | } 141 | 142 | # rm job 143 | internal_transfer_retry = 0 144 | assert await valid_client.submit_delete_job( 145 | machine="cluster1", 146 | target_path="/path/to/destination", 147 | job_name="mv-job", 148 | time="2", 149 | stage_out_job_id="35363851", 150 | account="project", 151 | ) == { 152 | "job_data_err": "", 153 | "job_data_out": "", 154 | "job_file": "/path/to/firecrest/internal_transfer_id/sbatch-job.sh", 155 | "job_file_err": "/path/to/firecrest/internal_transfer_id/job-35363861.err", 156 | "job_file_out": "/path/to/firecrest/internal_transfer_id/job-35363861.out", 157 | "jobid": 35363861, 158 | "result": "Job submitted", 159 | "system": "cluster1", 160 | } 161 | 162 | 163 | @pytest.mark.asyncio 164 | async def test_internal_transfer_invalid_machine(valid_client): 165 | with pytest.raises(firecrest.HeaderException): 166 | # mv job 167 | await valid_client.submit_move_job( 168 | machine="cluster2", 169 | source_path="/path/to/source", 170 | target_path="/path/to/destination", 171 | job_name="mv-job", 172 | time="2", 173 | stage_out_job_id="35363851", 174 | account="project", 175 | ) 176 | 177 | with pytest.raises(firecrest.HeaderException): 178 | # cp job 179 | await valid_client.submit_copy_job( 180 | machine="cluster2", 181 | source_path="/path/to/source", 182 | target_path="/path/to/destination", 183 | job_name="mv-job", 184 | time="2", 185 | stage_out_job_id="35363851", 186 | account="project", 187 | ) 188 | 189 | with pytest.raises(firecrest.HeaderException): 190 | # rsync job 191 | await valid_client.submit_rsync_job( 192 | machine="cluster2", 193 | source_path="/path/to/source", 194 | target_path="/path/to/destination", 195 | job_name="mv-job", 196 | time="2", 197 | stage_out_job_id="35363851", 198 | account="project", 199 | ) 200 | 201 | with pytest.raises(firecrest.HeaderException): 202 | # rm job 203 | await valid_client.submit_delete_job( 204 | machine="cluster2", 205 | target_path="/path/to/destination", 206 | job_name="mv-job", 207 | time="2", 208 | stage_out_job_id="35363851", 209 | account="project", 210 | ) 211 | 212 | 213 | @pytest.mark.asyncio 214 | async def test_internal_transfer_invalid_client(invalid_client): 215 | with pytest.raises(firecrest.UnauthorizedException): 216 | # mv job 217 | await invalid_client.submit_move_job( 218 | machine="cluster1", 219 | source_path="/path/to/source", 220 | target_path="/path/to/destination", 221 | job_name="mv-job", 222 | time="2", 223 | stage_out_job_id="35363851", 224 | account="project", 225 | ) 226 | 227 | with pytest.raises(firecrest.UnauthorizedException): 228 | # cp job 229 | await invalid_client.submit_copy_job( 230 | machine="cluster1", 231 | source_path="/path/to/source", 232 | target_path="/path/to/destination", 233 | job_name="mv-job", 234 | time="2", 235 | stage_out_job_id="35363851", 236 | account="project", 237 | ) 238 | 239 | with pytest.raises(firecrest.UnauthorizedException): 240 | # rsync job 241 | await invalid_client.submit_rsync_job( 242 | machine="cluster1", 243 | source_path="/path/to/source", 244 | target_path="/path/to/destination", 245 | job_name="mv-job", 246 | time="2", 247 | stage_out_job_id="35363851", 248 | account="project", 249 | ) 250 | 251 | with pytest.raises(firecrest.UnauthorizedException): 252 | # rm job 253 | await invalid_client.submit_delete_job( 254 | machine="cluster1", 255 | target_path="/path/to/destination", 256 | job_name="mv-job", 257 | time="2", 258 | stage_out_job_id="35363851", 259 | account="project", 260 | ) 261 | 262 | 263 | @pytest.mark.asyncio 264 | async def test_external_download(valid_client): 265 | global external_download_retry 266 | external_download_retry = 0 267 | valid_client.set_api_version("1.14.0") 268 | obj = await valid_client.external_download("cluster1", "/path/to/remote/source") 269 | assert isinstance(obj, firecrest.v1.AsyncExternalDownload) 270 | assert obj._task_id == "external_download_id" 271 | assert obj.client == valid_client 272 | 273 | 274 | @pytest.mark.asyncio 275 | async def test_external_download_legacy(valid_client): 276 | global external_download_retry 277 | external_download_retry = 0 278 | valid_client.set_api_version("1.13.0") 279 | obj = await valid_client.external_download("cluster1", "/path/to/remote/sourcelegacy") 280 | assert isinstance(obj, firecrest.v1.AsyncExternalDownload) 281 | assert obj._task_id == "external_download_id_legacy" 282 | assert obj.client == valid_client 283 | 284 | 285 | @pytest.mark.asyncio 286 | async def test_external_upload(valid_client): 287 | global external_upload_retry 288 | external_upload_retry = 0 289 | obj = await valid_client.external_upload( 290 | "cluster1", "/path/to/local/source", "/path/to/remote/destination" 291 | ) 292 | assert isinstance(obj, firecrest.v1.AsyncExternalUpload) 293 | assert obj._task_id == "external_upload_id" 294 | assert obj.client == valid_client 295 | -------------------------------------------------------------------------------- /tests/v2/common.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def clean_stdout(input): 5 | ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") 6 | return ansi_escape.sub("", input) 7 | -------------------------------------------------------------------------------- /tests/v2/context_v2.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) 5 | 6 | from firecrest.v2 import ( # noqa 7 | AsyncFirecrest, 8 | AsyncExternalDownload, 9 | AsyncExternalUpload, 10 | Firecrest, 11 | ExternalDownload, 12 | ExternalUpload, 13 | ) 14 | from firecrest.FirecrestException import UnexpectedStatusException 15 | -------------------------------------------------------------------------------- /tests/v2/handlers.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pytest 3 | import re 4 | 5 | from werkzeug.wrappers import Response 6 | from werkzeug.wrappers import Request 7 | 8 | 9 | def read_json_file(filename): 10 | with open(filename) as fp: 11 | data = json.load(fp) 12 | 13 | return data 14 | 15 | 16 | @pytest.fixture 17 | def auth_server(httpserver): 18 | httpserver.expect_request("/auth/token").respond_with_handler(auth_handler) 19 | return httpserver 20 | 21 | 22 | @pytest.fixture 23 | def fc_server(httpserver): 24 | httpserver.expect_request( 25 | re.compile("/status/.*"), method="GET" 26 | ).respond_with_handler(status_handler) 27 | 28 | for endpoint in ["ls", "view", "tail", "head", 29 | "checksum", "file", "stat"]: 30 | httpserver.expect_request( 31 | re.compile(rf"/filesystem/.*/{endpoint}"), method="GET" 32 | ).respond_with_handler(filesystem_handler) 33 | 34 | httpserver.expect_request( 35 | re.compile(r"/filesystem/.*/mkdir"), method="POST" 36 | ).respond_with_handler(filesystem_handler) 37 | 38 | for endpoint in ["chown", "chmod"]: 39 | httpserver.expect_request( 40 | re.compile(rf"/filesystem/.*/{endpoint}"), method="PUT" 41 | ).respond_with_handler(filesystem_handler) 42 | 43 | httpserver.expect_request( 44 | re.compile(r"/filesystem/.*/rm"), method="DELETE" 45 | ).respond_with_handler(filesystem_handler) 46 | 47 | for endpoint in ["jobs", "metadata"]: 48 | httpserver.expect_request( 49 | re.compile(rf"/compute/.*/{endpoint}"), method="GET" 50 | ).respond_with_handler(filesystem_handler) 51 | 52 | httpserver.expect_request( 53 | re.compile(r"/compute/.*/jobs"), method="POST" 54 | ).respond_with_handler(submit_handler) 55 | 56 | return httpserver 57 | 58 | def auth_handler(request): 59 | client_id = request.form["client_id"] 60 | client_secret = request.form["client_secret"] 61 | if client_id == "valid_id": 62 | if client_secret == "valid_secret": 63 | ret = { 64 | "access_token": "VALID_TOKEN", 65 | "expires_in": 15, 66 | "refresh_expires_in": 0, 67 | "token_type": "Bearer", 68 | "not-before-policy": 0, 69 | "scope": "profile firecrest email", 70 | } 71 | ret_status = 200 72 | elif client_secret == "valid_secret_2": 73 | ret = { 74 | "access_token": "token_2", 75 | "expires_in": 15, 76 | "refresh_expires_in": 0, 77 | "token_type": "Bearer", 78 | "not-before-policy": 0, 79 | "scope": "profile firecrest email", 80 | } 81 | ret_status = 200 82 | else: 83 | ret = { 84 | "error": "unauthorized_client", 85 | "error_description": "Invalid client secret", 86 | } 87 | ret_status = 400 88 | else: 89 | ret = { 90 | "error": "invalid_client", 91 | "error_description": "Invalid client credentials", 92 | } 93 | ret_status = 400 94 | 95 | return Response(json.dumps(ret), status=ret_status, content_type="application/json") 96 | 97 | 98 | def status_handler(request: Request): 99 | if request.headers["Authorization"] != "Bearer VALID_TOKEN": 100 | return Response( 101 | json.dumps({"message": "Bad token; invalid JSON"}), 102 | status=401, 103 | content_type="application/json", 104 | ) 105 | 106 | endpoint = request.url.split("/")[-1] 107 | data = read_json_file(f"v2/responses/{endpoint}.json") 108 | 109 | ret = data["response"] 110 | ret_status = data["status_code"] 111 | 112 | return Response(json.dumps(ret), 113 | status=ret_status, 114 | content_type="application/json") 115 | 116 | 117 | def filesystem_handler(request: Request): 118 | if request.headers["Authorization"] != "Bearer VALID_TOKEN": 119 | return Response( 120 | json.dumps({"message": "Bad token; invalid JSON"}), 121 | status=401, 122 | content_type="application/json", 123 | ) 124 | 125 | url, *params = request.url.split("?") 126 | 127 | endpoint = url.split("/")[-1] 128 | 129 | suffix = "" 130 | 131 | if endpoint == "head": 132 | if (request.args.get("path") == "/path/to/file" and 133 | request.args.get("skipEnding") == "false" and 134 | request.args.get("bytes") == "8"): 135 | suffix = "_bytes" 136 | 137 | if (request.args.get("path") == "/path/to/file" and 138 | request.args.get("skipEnding") == "true" and 139 | request.args.get("bytes") == "8"): 140 | suffix = "_bytes_exclude_trailing" 141 | 142 | if (request.args.get("path") == "/path/to/file" and 143 | request.args.get("skipEnding") == "false" and 144 | request.args.get("lines") == "4"): 145 | suffix = "_lines" 146 | 147 | if (request.args.get("path") == "/path/to/file" and 148 | request.args.get("skipEnding") == "true" and 149 | request.args.get("lines") == "4"): 150 | suffix = "_lines_exclude_trailing" 151 | 152 | if endpoint == "tail": 153 | if (request.args.get("path") == "/path/to/file" and 154 | request.args.get("skipBeginning") == "false" and 155 | request.args.get("bytes") == "8"): 156 | suffix = "_bytes" 157 | 158 | if (request.args.get("path") == "/path/to/file" and 159 | request.args.get("skipBeginning") == "true" and 160 | request.args.get("bytes") == "8"): 161 | suffix = "_bytes_exclude_beginning" 162 | 163 | if (request.args.get("path") == "/path/to/file" and 164 | request.args.get("skipBeginning") == "false" and 165 | request.args.get("lines") == "4"): 166 | suffix = "_lines" 167 | 168 | if (request.args.get("path") == "/path/to/file" and 169 | request.args.get("skipBeginning") == "true" and 170 | request.args.get("lines") == "4"): 171 | suffix = "_lines_exclude_beginning" 172 | 173 | if endpoint == "ls": 174 | if (request.args.get("path") == "/home/user" and 175 | request.args.get("showHidden") == "false" and 176 | request.args.get("recursive") == "false" and 177 | request.args.get("numericUid") == "false" and 178 | request.args.get("dereference") == "true"): 179 | suffix = "_dereference" 180 | 181 | if (request.args.get("path") == "/home/user" and 182 | request.args.get("showHidden") == "true" and 183 | request.args.get("recursive") == "false" and 184 | request.args.get("numericUid") == "false" and 185 | request.args.get("dereference") == "false"): 186 | suffix = "_hidden" 187 | 188 | if (request.args.get("path") == "/home/user" and 189 | request.args.get("showHidden") == "false" and 190 | request.args.get("recursive") == "true" and 191 | request.args.get("numericUid") == "false" and 192 | request.args.get("dereference") == "false"): 193 | suffix = "_recursive" 194 | 195 | if (request.args.get("path") == "/home/user" and 196 | request.args.get("showHidden") == "false" and 197 | request.args.get("recursive") == "false" and 198 | request.args.get("numericUid") == "true" and 199 | request.args.get("dereference") == "false"): 200 | suffix = "_uid" 201 | 202 | if (request.args.get("path") == "/invalid/path" and 203 | request.args.get("showHidden") == "false" and 204 | request.args.get("recursive") == "false" and 205 | request.args.get("numericUid") == "false" and 206 | request.args.get("dereference") == "false"): 207 | suffix = "_invalid_path" 208 | 209 | if endpoint == "stat": 210 | if (request.args.get("path") == "/home/user/file" and 211 | request.args.get("dereference") == "true"): 212 | suffix = "_dereference" 213 | 214 | if endpoint == "chown": 215 | data = json.loads(request.get_data()) 216 | if data == { 217 | 'path': '/home/test1/xxx', 218 | 'owner': 'test1', 219 | 'group': 'users' 220 | }: 221 | suffix = "_not_permitted" 222 | 223 | if endpoint == "jobs": 224 | endpoint = "job" 225 | suffix = "_info" 226 | 227 | if endpoint == "1": 228 | endpoint = "job" 229 | suffix = "_info" 230 | 231 | if endpoint == "metadata": 232 | endpoint = "job" 233 | suffix = "_metadata" 234 | 235 | data = read_json_file(f"v2/responses/{endpoint}{suffix}.json") 236 | 237 | ret = data["response"] 238 | ret_status = data["status_code"] 239 | 240 | return Response(json.dumps(ret), 241 | status=ret_status, 242 | content_type="application/json") 243 | 244 | 245 | def submit_handler(request: Request): 246 | if request.headers["Authorization"] != "Bearer VALID_TOKEN": 247 | return Response( 248 | json.dumps({"message": "Bad token; invalid JSON"}), 249 | status=401, 250 | content_type="application/json", 251 | ) 252 | 253 | url, *params = request.url.split("?") 254 | 255 | endpoint = url.split("/")[-1] 256 | 257 | suffix = "" 258 | 259 | if endpoint == "jobs": 260 | endpoint = "job" 261 | suffix = "_submit" 262 | 263 | data = read_json_file(f"v2/responses/{endpoint}{suffix}.json") 264 | 265 | ret = data["response"] 266 | ret_status = data["status_code"] 267 | 268 | return Response(json.dumps(ret), 269 | status=ret_status, 270 | content_type="application/json") 271 | -------------------------------------------------------------------------------- /tests/v2/responses/checksum.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "algorithm": "SHA256", 6 | "checksum": "67149111d45cf106eb92ab5be7ec08179bddea7426ddde7cfe0ae68a7cffce74" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tests/v2/responses/chmod.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "name": "/home/test1/xxx", 6 | "type": "-", 7 | "linkTarget": null, 8 | "user": "test1", 9 | "group": "users", 10 | "permissions": "rwxrwxrwx", 11 | "lastModified": "2024-10-24T15:00:01", 12 | "size": "0" 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/v2/responses/chown.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "name": "/home/test1/xxx", 6 | "type": "-", 7 | "linkTarget": null, 8 | "user": "test1", 9 | "group": "users", 10 | "permissions": "rwxrwxrwx", 11 | "lastModified": "2024-10-24T15:00:01", 12 | "size": "0" 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/v2/responses/chown_not_permitted.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 403, 3 | "response": { 4 | "errorType": "error", 5 | "message": "chown: changing ownership of '/home/test1/xxx': Operation not permitted", 6 | "data": null, 7 | "user": "test1", 8 | "authHeader": "Bearer" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /tests/v2/responses/compress.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 204, 3 | "response": null 4 | } 5 | -------------------------------------------------------------------------------- /tests/v2/responses/delete.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 204, 3 | "response": null 4 | } 5 | -------------------------------------------------------------------------------- /tests/v2/responses/extract.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 204, 3 | "response": null 4 | } 5 | -------------------------------------------------------------------------------- /tests/v2/responses/file.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": "ASCII text" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tests/v2/responses/head.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n", 6 | "contentType": "lines", 7 | "startPosition": 0, 8 | "endPosition": 10 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/head_bytes.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "1\n2\n3\n4", 6 | "contentType": "bytes", 7 | "startPosition": 0, 8 | "endPosition": 7 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/head_bytes_exclude_trailing.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "1\n2\n3\n4\n5\n6\n7\n8\n9\n10", 6 | "contentType": "bytes", 7 | "startPosition": 0, 8 | "endPosition": -7 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/head_lines.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "1\n2\n3\n", 6 | "contentType": "lines", 7 | "startPosition": 0, 8 | "endPosition": 3 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/head_lines_exclude_trailing.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "1\n2\n3\n4\n5\n6\n7\n8\n9\n", 6 | "contentType": "lines", 7 | "startPosition": 0, 8 | "endPosition": -3 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/job_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "jobs": [ 5 | { 6 | "account": "test", 7 | "allocationNodes": 1, 8 | "cluster": "cluster", 9 | "exitCode": { 10 | "status": [ 11 | "ERROR" 12 | ], 13 | "returnCode": 8 14 | }, 15 | "group": "users", 16 | "jobId": 1, 17 | "name": "allocation", 18 | "nodes": "localhost", 19 | "partition": "part01", 20 | "priority": 1, 21 | "killRequestUser": "", 22 | "state": { 23 | "current": [ 24 | "FAILED" 25 | ], 26 | "reason": "None" 27 | }, 28 | "time": { 29 | "elapsed": 0, 30 | "start": 1730968675, 31 | "end": 1730968675, 32 | "suspended": 0, 33 | "limit": 7200, 34 | "submission": 1730968674 35 | }, 36 | "user": "test1", 37 | "workingDirectory": "/home/test1" 38 | } 39 | ] 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /tests/v2/responses/job_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "jobs": [ 5 | { 6 | "jobId": 26, 7 | "script": "#!/bin/sh\npwd", 8 | "standardInput": "/dev/null", 9 | "standardOutput": "/home/test1/slurm-26.out", 10 | "standardError": "/home/test1/slurm-26.out" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tests/v2/responses/job_submit.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 201, 3 | "response": { 4 | "jobId": 27 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tests/v2/responses/ls.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2022-03-15T11:33:15", 13 | "size": "4096" 14 | } 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_dereference.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2024-10-23T09:06:00", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": "link_to_file", 17 | "type": "-", 18 | "linkTarget": null, 19 | "user": "root", 20 | "group": "root", 21 | "permissions": "rw-r--r--", 22 | "lastModified": "2024-10-23T09:06:00", 23 | "size": "0" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_hidden.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2022-03-15T11:33:15", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": ".bashrc", 17 | "type": "-", 18 | "linkTarget": null, 19 | "user": "test1", 20 | "group": "users", 21 | "permissions": "rw-r--r--", 22 | "lastModified": "2022-05-07T15:11:50", 23 | "size": "1177" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_home.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2022-03-15T11:33:15", 13 | "size": "4096" 14 | } 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_home_dereference.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2024-10-23T09:06:00", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": "link_to_file", 17 | "type": "-", 18 | "linkTarget": null, 19 | "user": "root", 20 | "group": "root", 21 | "permissions": "rw-r--r--", 22 | "lastModified": "2024-10-23T09:06:00", 23 | "size": "0" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_home_hidden.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2022-03-15T11:33:15", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": ".bashrc", 17 | "type": "-", 18 | "linkTarget": null, 19 | "user": "test1", 20 | "group": "users", 21 | "permissions": "rw-r--r--", 22 | "lastModified": "2022-05-07T15:11:50", 23 | "size": "1177" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_home_recursive.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "/home/test1/bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2024-10-23T09:06:00", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": "/home/test1/bin/file", 17 | "type": "-", 18 | "linkTarget": null, 19 | "user": "root", 20 | "group": "root", 21 | "permissions": "rw-r--r--", 22 | "lastModified": "2024-10-23T09:06:00", 23 | "size": "0" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_home_uid.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "1000", 10 | "group": "100", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2024-10-23T09:06:00", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": "link_to_file", 17 | "type": "l", 18 | "linkTarget": "bin/file", 19 | "user": "0", 20 | "group": "0", 21 | "permissions": "rwxrwxrwx", 22 | "lastModified": "2024-10-23T09:10:04", 23 | "size": "8" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_invalid_path.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 404, 3 | "response": { 4 | "errorType": "error", 5 | "message": "ls: cannot access '/invalid/path': No such file or directory", 6 | "data": null, 7 | "user": "test1", 8 | "authHeader": "Bearer token" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_recursive.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "/home/test1/bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "test1", 10 | "group": "users", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2024-10-23T09:06:00", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": "/home/test1/bin/file", 17 | "type": "-", 18 | "linkTarget": null, 19 | "user": "root", 20 | "group": "root", 21 | "permissions": "rw-r--r--", 22 | "lastModified": "2024-10-23T09:06:00", 23 | "size": "0" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/ls_uid.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": [ 5 | { 6 | "name": "bin", 7 | "type": "d", 8 | "linkTarget": null, 9 | "user": "1000", 10 | "group": "100", 11 | "permissions": "rwxr-xr-x", 12 | "lastModified": "2024-10-23T09:06:00", 13 | "size": "4096" 14 | }, 15 | { 16 | "name": "link_to_file", 17 | "type": "l", 18 | "linkTarget": "bin/file", 19 | "user": "0", 20 | "group": "0", 21 | "permissions": "rwxrwxrwx", 22 | "lastModified": "2024-10-23T09:10:04", 23 | "size": "8" 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/v2/responses/mkdir.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 201, 3 | "response": { 4 | "output": { 5 | "name": "/home/fireuser/new_dir", 6 | "type": "d", 7 | "linkTarget": "None", 8 | "user": "fireuser", 9 | "group": "users", 10 | "permissions": "rwxr-xr-x", 11 | "lastModified": "2024-12-19T10:29:59", 12 | "size": "4096" 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/v2/responses/nodes.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "nodes": [ 5 | { 6 | "sockets": 2, 7 | "cores": 1, 8 | "threads": 1, 9 | "cpus": 2, 10 | "cpuLoad": 229.0, 11 | "freeMemory": null, 12 | "features": [ 13 | "f7t" 14 | ], 15 | "name": "localhost", 16 | "address": "localhost", 17 | "hostname": "localhost", 18 | "state": [ 19 | "IDLE" 20 | ], 21 | "partitions": [ 22 | "part01", 23 | "part02", 24 | "xfer" 25 | ], 26 | "weight": 1, 27 | "slurmdVersion": null, 28 | "allocMemory": 0, 29 | "allocCpus": 0, 30 | "idleCpus": null 31 | } 32 | ] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tests/v2/responses/partitions.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "partitions": [ 5 | { 6 | "partitionName": "part01", 7 | "cpus": 2, 8 | "totalNodes": 1, 9 | "state": [ 10 | "UP" 11 | ] 12 | }, 13 | { 14 | "partitionName": "part02", 15 | "cpus": 2, 16 | "totalNodes": 1, 17 | "state": [ 18 | "UP" 19 | ] 20 | }, 21 | { 22 | "partitionName": "xfer", 23 | "cpus": 2, 24 | "totalNodes": 1, 25 | "state": [ 26 | "UP" 27 | ] 28 | } 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tests/v2/responses/reservations.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "reservations": [ 5 | { 6 | "reservationName": "root_1", 7 | "nodes": "localhost", 8 | "endTime": 1729173600, 9 | "startTime": 1729159200, 10 | "features": "" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tests/v2/responses/rm.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 204, 3 | "response": null 4 | } 5 | -------------------------------------------------------------------------------- /tests/v2/responses/stat.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "mode": 33188, 6 | "ino": 2299756, 7 | "dev": 76, 8 | "nlink": 1, 9 | "uid": 0, 10 | "gid": 0, 11 | "size": 27, 12 | "atime": 1729771818, 13 | "ctime": 1729676630, 14 | "mtime": 1729676630 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/v2/responses/stat_dereference.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "mode": 16877, 6 | "ino": 2297286, 7 | "dev": 76, 8 | "nlink": 1, 9 | "uid": 1000, 10 | "gid": 100, 11 | "size": 4096, 12 | "atime": 1729674368, 13 | "ctime": 1729674360, 14 | "mtime": 1729674360 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/v2/responses/symlink.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 201, 3 | "response": { 4 | "output": { 5 | "name": "/home/fireuser/upload_folder/dom-60s.sh", 6 | "type": "-", 7 | "linkTarget": null, 8 | "user": "fireuser", 9 | "group": "users", 10 | "permissions": "rw-r--r--", 11 | "lastModified": "2024-12-19T11:00:44", 12 | "size": "0" 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/v2/responses/systems.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "systems": [ 5 | { 6 | "name": "cluster-api", 7 | "host": "192.168.240.2", 8 | "sshPort": 22, 9 | "sshCertEmbeddedCmd": true, 10 | "scheduler": { 11 | "type": "slurm", 12 | "version": "24.05.0", 13 | "apiUrl": "http://192.168.240.2:6820", 14 | "apiVersion": "0.0.40" 15 | }, 16 | "health": { 17 | "lastChecked": "2024-10-04T14:39:29.092143Z", 18 | "latency": 0.006885528564453125, 19 | "healthy": true, 20 | "message": null, 21 | "nodes": { 22 | "available": 1, 23 | "total": 1 24 | } 25 | }, 26 | "probing": { 27 | "interval": 60, 28 | "timeout": 10, 29 | "healthyLatency": 1.5, 30 | "healthyLoad": 0.8, 31 | "startupGracePeriod": 300 32 | }, 33 | "fileSystems": [ 34 | { 35 | "path": "/home", 36 | "dataType": "users", 37 | "defaultWorkDir": true 38 | }, 39 | { 40 | "path": "/store", 41 | "dataType": "store", 42 | "defaultWorkDir": false 43 | }, 44 | { 45 | "path": "/archive", 46 | "dataType": "archive", 47 | "defaultWorkDir": false 48 | } 49 | ], 50 | "datatransferJobsDirectives": [ 51 | "#SBATCH --constraint=mc", 52 | "#SBATCH --nodes=1", 53 | "#SBATCH --time=0-00:15:00" 54 | ], 55 | "timeouts": { 56 | "sshConnection": 5, 57 | "sshLogin": 5, 58 | "sshCommandExecution": 5 59 | } 60 | }, 61 | { 62 | "name": "cluster-ssh", 63 | "host": "192.168.240.2", 64 | "sshPort": 22, 65 | "sshCertEmbeddedCmd": true, 66 | "scheduler": { 67 | "type": "slurm", 68 | "version": "24.05.0", 69 | "apiUrl": null, 70 | "apiVersion": null 71 | }, 72 | "health": { 73 | "lastChecked": "2024-10-04T14:39:29.696364Z", 74 | "latency": 0.6117508411407471, 75 | "healthy": true, 76 | "message": null, 77 | "nodes": { 78 | "available": 1, 79 | "total": 1 80 | } 81 | }, 82 | "probing": { 83 | "interval": 60, 84 | "timeout": 10, 85 | "healthyLatency": 1.5, 86 | "healthyLoad": 0.8, 87 | "startupGracePeriod": 300 88 | }, 89 | "fileSystems": [ 90 | { 91 | "path": "/home", 92 | "dataType": "users", 93 | "defaultWorkDir": true 94 | }, 95 | { 96 | "path": "/store", 97 | "dataType": "store", 98 | "defaultWorkDir": false 99 | }, 100 | { 101 | "path": "/scratch", 102 | "dataType": "scratch", 103 | "defaultWorkDir": false 104 | } 105 | ], 106 | "datatransferJobsDirectives": [], 107 | "timeouts": { 108 | "sshConnection": 5, 109 | "sshLogin": 5, 110 | "sshCommandExecution": 5 111 | } 112 | } 113 | ] 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /tests/v2/responses/tail.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n", 6 | "contentType": "lines", 7 | "startPosition": 10, 8 | "endPosition": -1 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/tail_bytes.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "\n11\n12\n", 6 | "contentType": "bytes", 7 | "startPosition": -7, 8 | "endPosition": -1 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/tail_bytes_exclude_beginning.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "4\n5\n6\n7\n8\n9\n10\n11\n12\n", 6 | "contentType": "bytes", 7 | "startPosition": 7, 8 | "endPosition": -1 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/tail_lines.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "10\n11\n12\n", 6 | "contentType": "lines", 7 | "startPosition": -3, 8 | "endPosition": -1 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/tail_lines_exclude_beginning.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "content": "3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n", 6 | "contentType": "lines", 7 | "startPosition": 3, 8 | "endPosition": -1 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/v2/responses/userinfo.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "user": { 5 | "id": "1000", 6 | "name": "fireuser" 7 | }, 8 | "group": { 9 | "id": "100", 10 | "name": "users" 11 | }, 12 | "groups": [ 13 | { 14 | "id": "100", 15 | "name": "users" 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /tests/v2/responses/view.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_code": 200, 3 | "response": { 4 | "output": { 5 | "output": "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tests/v2/test_v2_async.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pytest 3 | import re 4 | 5 | from context_v2 import AsyncFirecrest, UnexpectedStatusException 6 | from werkzeug.wrappers import Response 7 | from werkzeug.wrappers import Request 8 | from handlers import (fc_server, 9 | filesystem_handler, 10 | read_json_file, 11 | status_handler, 12 | submit_handler) 13 | 14 | 15 | @pytest.fixture 16 | def valid_client(fc_server): 17 | class ValidAuthorization: 18 | def get_access_token(self): 19 | return "VALID_TOKEN" 20 | 21 | return AsyncFirecrest( 22 | firecrest_url=fc_server.url_for("/"), 23 | authorization=ValidAuthorization() 24 | ) 25 | 26 | 27 | @pytest.fixture 28 | def invalid_client(fc_server): 29 | class InvalidAuthorization: 30 | def get_access_token(self): 31 | return "INVALID_TOKEN" 32 | 33 | return AsyncFirecrest( 34 | firecrest_url=fc_server.url_for("/"), 35 | authorization=InvalidAuthorization() 36 | ) 37 | 38 | 39 | @pytest.mark.asyncio 40 | async def test_systems(valid_client): 41 | data = read_json_file("v2/responses/systems.json") 42 | resp = await valid_client.systems() 43 | assert resp == data["response"]["systems"] 44 | 45 | 46 | @pytest.mark.asyncio 47 | async def test_partitions(valid_client): 48 | data = read_json_file("v2/responses/partitions.json") 49 | resp = await valid_client.partitions("cluster") 50 | assert resp == data["response"]["partitions"] 51 | 52 | 53 | @pytest.mark.asyncio 54 | async def test_nodes(valid_client): 55 | data = read_json_file("v2/responses/nodes.json") 56 | resp = await valid_client.nodes("cluster") 57 | assert resp == data["response"]["nodes"] 58 | 59 | 60 | @pytest.mark.asyncio 61 | async def test_reservations(valid_client): 62 | data = read_json_file("v2/responses/reservations.json") 63 | resp = await valid_client.reservations("cluster") 64 | assert resp == data["response"]["reservations"] 65 | 66 | 67 | @pytest.mark.asyncio 68 | async def test_userinfo(valid_client): 69 | data = read_json_file("v2/responses/userinfo.json") 70 | resp = await valid_client.userinfo("cluster") 71 | assert resp == data["response"] 72 | 73 | 74 | @pytest.mark.asyncio 75 | async def test_head(valid_client): 76 | data = read_json_file("v2/responses/head.json") 77 | resp = await valid_client.head("cluster", "/path/to/file") 78 | assert resp == data["response"]["output"] 79 | 80 | 81 | @pytest.mark.asyncio 82 | async def test_head_bytes(valid_client): 83 | data = read_json_file("v2/responses/head_bytes.json") 84 | resp = await valid_client.head("cluster", "/path/to/file", num_bytes=8) 85 | assert resp == data["response"]["output"] 86 | 87 | 88 | @pytest.mark.asyncio 89 | async def test_head_bytes_exclude_trailing(valid_client): 90 | data = read_json_file("v2/responses/head_bytes_exclude_trailing.json") 91 | resp = await valid_client.head("cluster", "/path/to/file", 92 | num_bytes=8, exclude_trailing=True) 93 | assert resp == data["response"]["output"] 94 | 95 | 96 | @pytest.mark.asyncio 97 | async def test_head_lines(valid_client): 98 | data = read_json_file("v2/responses/head_lines.json") 99 | resp = await valid_client.head("cluster", "/path/to/file", 100 | num_lines=4) 101 | assert resp == data["response"]["output"] 102 | 103 | 104 | @pytest.mark.asyncio 105 | async def test_head_lines_exclude_trailing(valid_client): 106 | data = read_json_file("v2/responses/head_lines_exclude_trailing.json") 107 | resp = await valid_client.head("cluster", "/path/to/file", 108 | exclude_trailing=True, num_lines=4) 109 | assert resp == data["response"]["output"] 110 | 111 | 112 | @pytest.mark.asyncio 113 | async def test_head_lines_and_bytes(valid_client): 114 | with pytest.raises(ValueError) as excinfo: 115 | await valid_client.head("cluster", "/path/to/file", num_bytes=8, 116 | num_lines=4) 117 | 118 | assert str(excinfo.value) == ( 119 | "You cannot specify both `num_bytes` and `num_lines`." 120 | ) 121 | 122 | 123 | @pytest.mark.asyncio 124 | async def test_tail(valid_client): 125 | data = read_json_file("v2/responses/tail.json") 126 | resp = await valid_client.tail("cluster", "/path/to/file") 127 | assert resp == data["response"]["output"] 128 | 129 | 130 | @pytest.mark.asyncio 131 | async def test_tail_bytes(valid_client): 132 | data = read_json_file("v2/responses/tail_bytes.json") 133 | resp = await valid_client.tail("cluster", "/path/to/file", num_bytes=8) 134 | assert resp == data["response"]["output"] 135 | 136 | 137 | @pytest.mark.asyncio 138 | async def test_tail_bytes_exclude_beginning(valid_client): 139 | data = read_json_file("v2/responses/tail_bytes_exclude_beginning.json") 140 | resp = await valid_client.tail("cluster", "/path/to/file", 141 | num_bytes=8, exclude_beginning=True) 142 | assert resp == data["response"]["output"] 143 | 144 | 145 | @pytest.mark.asyncio 146 | async def test_tail_lines(valid_client): 147 | data = read_json_file("v2/responses/tail_lines.json") 148 | resp = await valid_client.tail("cluster", "/path/to/file", 149 | num_lines=4) 150 | assert resp == data["response"]["output"] 151 | 152 | 153 | @pytest.mark.asyncio 154 | async def test_tail_lines_exclude_beginning(valid_client): 155 | data = read_json_file("v2/responses/tail_lines_exclude_beginning.json") 156 | resp = await valid_client.tail("cluster", "/path/to/file", 157 | exclude_beginning=True, num_lines=4) 158 | assert resp == data["response"]["output"] 159 | 160 | 161 | @pytest.mark.asyncio 162 | async def test_tail_lines_and_bytes(valid_client): 163 | with pytest.raises(ValueError) as excinfo: 164 | await valid_client.tail("cluster", "/path/to/file", num_bytes=8, 165 | num_lines=4) 166 | 167 | assert str(excinfo.value) == ( 168 | "You cannot specify both `num_bytes` and `num_lines`." 169 | ) 170 | 171 | 172 | @pytest.mark.asyncio 173 | async def test_ls(valid_client): 174 | data = read_json_file("v2/responses/ls.json") 175 | resp = await valid_client.list_files("cluster", "/home/user") 176 | assert resp == data["response"]["output"] 177 | 178 | 179 | @pytest.mark.asyncio 180 | async def test_ls_dereference(valid_client): 181 | data = read_json_file("v2/responses/ls_dereference.json") 182 | resp = await valid_client.list_files("cluster", "/home/user", 183 | dereference=True) 184 | assert resp == data["response"]["output"] 185 | 186 | 187 | @pytest.mark.asyncio 188 | async def test_ls_hidden(valid_client): 189 | data = read_json_file("v2/responses/ls_hidden.json") 190 | resp = await valid_client.list_files("cluster", "/home/user", 191 | show_hidden=True) 192 | assert resp == data["response"]["output"] 193 | 194 | 195 | @pytest.mark.asyncio 196 | async def test_ls_recursive(valid_client): 197 | data = read_json_file("v2/responses/ls_recursive.json") 198 | resp = await valid_client.list_files("cluster", "/home/user", 199 | recursive=True) 200 | assert resp == data["response"]["output"] 201 | 202 | 203 | @pytest.mark.asyncio 204 | async def test_ls_uid(valid_client): 205 | data = read_json_file("v2/responses/ls_uid.json") 206 | resp = await valid_client.list_files("cluster", "/home/user", 207 | numeric_uid=True) 208 | assert resp == data["response"]["output"] 209 | 210 | 211 | @pytest.mark.asyncio 212 | async def test_ls_invalid_path(valid_client): 213 | data = read_json_file("v2/responses/ls_invalid_path.json") 214 | with pytest.raises(UnexpectedStatusException) as excinfo: 215 | await valid_client.list_files("cluster", "/invalid/path") 216 | 217 | byte_content = excinfo.value.responses[-1].content 218 | decoded_string = byte_content.decode('utf-8') 219 | response_dict = json.loads(decoded_string) 220 | message = response_dict["message"] 221 | 222 | assert str(message) == ( 223 | "ls: cannot access '/invalid/path': No such file or directory" 224 | ) 225 | 226 | 227 | @pytest.mark.asyncio 228 | async def test_view(valid_client): 229 | data = read_json_file("v2/responses/view.json") 230 | resp = await valid_client.view("cluster", "/home/user/file") 231 | assert resp == data["response"]["output"] 232 | 233 | 234 | 235 | @pytest.mark.asyncio 236 | async def test_stat(valid_client): 237 | data = read_json_file("v2/responses/stat.json") 238 | resp = await valid_client.stat("cluster", "/home/user/file") 239 | assert resp == data["response"]["output"] 240 | 241 | 242 | @pytest.mark.asyncio 243 | async def test_stat_dereference(valid_client): 244 | data = read_json_file("v2/responses/stat_dereference.json") 245 | resp = await valid_client.stat("cluster", "/home/user/file", 246 | dereference=True) 247 | assert resp == data["response"]["output"] 248 | 249 | 250 | @pytest.mark.asyncio 251 | async def test_file_type(valid_client): 252 | data = read_json_file("v2/responses/file.json") 253 | resp = await valid_client.file_type("cluster", "/home/user/file") 254 | assert resp == data["response"]["output"] 255 | 256 | 257 | @pytest.mark.asyncio 258 | async def test_checksum(valid_client): 259 | data = read_json_file("v2/responses/checksum.json") 260 | resp = await valid_client.checksum("cluster", "/home/user/file") 261 | assert resp == data["response"]["output"] 262 | 263 | 264 | @pytest.mark.asyncio 265 | async def test_mkdir(valid_client): 266 | data = read_json_file("v2/responses/mkdir.json") 267 | resp = await valid_client.mkdir("cluster", "/home/user/file") 268 | assert resp == data["response"]["output"] 269 | 270 | 271 | @pytest.mark.asyncio 272 | async def test_chown(valid_client): 273 | data = read_json_file("v2/responses/chown.json") 274 | resp = await valid_client.chown("cluster", "/home/user/file", 275 | "test1", "users") 276 | assert resp == data["response"]["output"] 277 | 278 | 279 | @pytest.mark.asyncio 280 | async def test_chown_not_permitted(valid_client): 281 | data = read_json_file("v2/responses/chown_not_permitted.json") 282 | with pytest.raises(UnexpectedStatusException) as excinfo: 283 | await valid_client.chown("cluster", "/home/test1/xxx", 284 | "test1", "users") 285 | 286 | assert str(excinfo.value) == ( 287 | f"last request: 403 {data['response']}: expected status 200" 288 | ) 289 | 290 | 291 | @pytest.mark.asyncio 292 | async def test_chmod(valid_client): 293 | data = read_json_file("v2/responses/chmod.json") 294 | resp = await valid_client.chmod("cluster", "/home/user/xxx", 295 | "777") 296 | assert resp == data["response"]["output"] 297 | 298 | 299 | @pytest.mark.asyncio 300 | async def test_rm(valid_client): 301 | data = read_json_file("v2/responses/rm.json") 302 | resp = await valid_client.rm("cluster", "/home/user/file") 303 | assert resp == data["response"]# ["output"] 304 | 305 | 306 | @pytest.mark.asyncio 307 | async def test_job_info(valid_client): 308 | data = read_json_file("v2/responses/job_info.json") 309 | resp = await valid_client.job_info("cluster") 310 | assert resp == data["response"]["jobs"] 311 | 312 | 313 | @pytest.mark.asyncio 314 | async def test_job_info_jobid(valid_client): 315 | data = read_json_file("v2/responses/job_info.json") 316 | resp = await valid_client.job_info("cluster", "1") 317 | assert resp == data["response"]["jobs"] 318 | 319 | 320 | @pytest.mark.asyncio 321 | async def test_job_metadata(valid_client): 322 | data = read_json_file("v2/responses/job_metadata.json") 323 | resp = await valid_client.job_metadata("cluster", "1") 324 | assert resp == data["response"]["jobs"] 325 | 326 | 327 | @pytest.mark.asyncio 328 | async def test_job_submit(valid_client): 329 | data = read_json_file("v2/responses/job_submit.json") 330 | resp = await valid_client.submit("cluster", "/path/to/dir", 331 | script_str="...") 332 | assert resp == data["response"] 333 | 334 | 335 | @pytest.mark.asyncio 336 | async def test_job_submit_no_script(valid_client): 337 | with pytest.raises(ValueError) as excinfo: 338 | await valid_client.submit("cluster", "/path/to/dir") 339 | 340 | assert str(excinfo.value) == ( 341 | "Exactly one of the arguments `script_str` or " 342 | "`script_local_path` must be set." 343 | ) 344 | -------------------------------------------------------------------------------- /tests/v2/test_v2_sync.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pytest 3 | import re 4 | 5 | from context_v2 import Firecrest, UnexpectedStatusException 6 | from werkzeug.wrappers import Response 7 | from werkzeug.wrappers import Request 8 | from handlers import (fc_server, 9 | filesystem_handler, 10 | read_json_file, 11 | status_handler, 12 | submit_handler) 13 | 14 | 15 | @pytest.fixture 16 | def valid_client(fc_server): 17 | class ValidAuthorization: 18 | def get_access_token(self): 19 | return "VALID_TOKEN" 20 | 21 | return Firecrest( 22 | firecrest_url=fc_server.url_for("/"), 23 | authorization=ValidAuthorization() 24 | ) 25 | 26 | 27 | @pytest.fixture 28 | def invalid_client(fc_server): 29 | class InvalidAuthorization: 30 | def get_access_token(self): 31 | return "INVALID_TOKEN" 32 | 33 | return Firecrest( 34 | firecrest_url=fc_server.url_for("/"), 35 | authorization=InvalidAuthorization() 36 | ) 37 | 38 | 39 | def test_systems(valid_client): 40 | data = read_json_file("v2/responses/systems.json") 41 | resp = valid_client.systems() 42 | assert resp == data["response"]["systems"] 43 | 44 | 45 | def test_partitions(valid_client): 46 | data = read_json_file("v2/responses/partitions.json") 47 | resp = valid_client.partitions("cluster") 48 | assert resp == data["response"]["partitions"] 49 | 50 | 51 | def test_nodes(valid_client): 52 | data = read_json_file("v2/responses/nodes.json") 53 | resp = valid_client.nodes("cluster") 54 | assert resp == data["response"]["nodes"] 55 | 56 | 57 | def test_reservations(valid_client): 58 | data = read_json_file("v2/responses/reservations.json") 59 | resp = valid_client.reservations("cluster") 60 | assert resp == data["response"]["reservations"] 61 | 62 | 63 | def test_userinfo(valid_client): 64 | data = read_json_file("v2/responses/userinfo.json") 65 | resp = valid_client.userinfo("cluster") 66 | assert resp == data["response"] 67 | 68 | 69 | def test_head(valid_client): 70 | data = read_json_file("v2/responses/head.json") 71 | resp = valid_client.head("cluster", "/path/to/file") 72 | assert resp == data["response"]["output"] 73 | 74 | 75 | def test_head_bytes(valid_client): 76 | data = read_json_file("v2/responses/head_bytes.json") 77 | resp = valid_client.head("cluster", "/path/to/file", num_bytes=8) 78 | assert resp == data["response"]["output"] 79 | 80 | 81 | def test_head_bytes_exclude_trailing(valid_client): 82 | data = read_json_file("v2/responses/head_bytes_exclude_trailing.json") 83 | resp = valid_client.head("cluster", "/path/to/file", 84 | num_bytes=8, exclude_trailing=True) 85 | assert resp == data["response"]["output"] 86 | 87 | 88 | def test_head_lines(valid_client): 89 | data = read_json_file("v2/responses/head_lines.json") 90 | resp = valid_client.head("cluster", "/path/to/file", 91 | num_lines=4) 92 | assert resp == data["response"]["output"] 93 | 94 | 95 | def test_head_lines_exclude_trailing(valid_client): 96 | data = read_json_file("v2/responses/head_lines_exclude_trailing.json") 97 | resp = valid_client.head("cluster", "/path/to/file", 98 | exclude_trailing=True, num_lines=4) 99 | assert resp == data["response"]["output"] 100 | 101 | def test_head_lines_and_bytes(valid_client): 102 | with pytest.raises(ValueError) as excinfo: 103 | valid_client.head("cluster", "/path/to/file", num_bytes=8, 104 | num_lines=4) 105 | 106 | assert str(excinfo.value) == ( 107 | "You cannot specify both `num_bytes` and `num_lines`." 108 | ) 109 | 110 | 111 | def test_tail(valid_client): 112 | data = read_json_file("v2/responses/tail.json") 113 | resp = valid_client.tail("cluster", "/path/to/file") 114 | assert resp == data["response"]["output"] 115 | 116 | 117 | def test_tail_bytes(valid_client): 118 | data = read_json_file("v2/responses/tail_bytes.json") 119 | resp = valid_client.tail("cluster", "/path/to/file", num_bytes=8) 120 | assert resp == data["response"]["output"] 121 | 122 | 123 | def test_tail_bytes_exclude_beginning(valid_client): 124 | data = read_json_file("v2/responses/tail_bytes_exclude_beginning.json") 125 | resp = valid_client.tail("cluster", "/path/to/file", 126 | num_bytes=8, exclude_beginning=True) 127 | assert resp == data["response"]["output"] 128 | 129 | 130 | def test_tail_lines(valid_client): 131 | data = read_json_file("v2/responses/tail_lines.json") 132 | resp = valid_client.tail("cluster", "/path/to/file", 133 | num_lines=4) 134 | assert resp == data["response"]["output"] 135 | 136 | 137 | def test_tail_lines_exclude_beginning(valid_client): 138 | data = read_json_file("v2/responses/tail_lines_exclude_beginning.json") 139 | resp = valid_client.tail("cluster", "/path/to/file", 140 | exclude_beginning=True, num_lines=4) 141 | assert resp == data["response"]["output"] 142 | 143 | def test_tail_lines_and_bytes(valid_client): 144 | with pytest.raises(ValueError) as excinfo: 145 | valid_client.tail("cluster", "/path/to/file", num_bytes=8, 146 | num_lines=4) 147 | 148 | assert str(excinfo.value) == ( 149 | "You cannot specify both `num_bytes` and `num_lines`." 150 | ) 151 | 152 | 153 | def test_ls(valid_client): 154 | data = read_json_file("v2/responses/ls.json") 155 | resp = valid_client.list_files("cluster", "/home/user") 156 | assert resp == data["response"]["output"] 157 | 158 | 159 | def test_ls_dereference(valid_client): 160 | data = read_json_file("v2/responses/ls_dereference.json") 161 | resp = valid_client.list_files("cluster", "/home/user", 162 | dereference=True) 163 | assert resp == data["response"]["output"] 164 | 165 | 166 | def test_ls_hidden(valid_client): 167 | data = read_json_file("v2/responses/ls_hidden.json") 168 | resp = valid_client.list_files("cluster", "/home/user", 169 | show_hidden=True) 170 | 171 | assert resp == data["response"]["output"] 172 | 173 | 174 | def test_ls_recursive(valid_client): 175 | data = read_json_file("v2/responses/ls_recursive.json") 176 | resp = valid_client.list_files("cluster", "/home/user", 177 | recursive=True) 178 | 179 | assert resp == data["response"]["output"] 180 | 181 | 182 | def test_ls_uid(valid_client): 183 | data = read_json_file("v2/responses/ls_uid.json") 184 | resp = valid_client.list_files("cluster", "/home/user", 185 | numeric_uid=True) 186 | 187 | assert resp == data["response"]["output"] 188 | 189 | 190 | def test_ls_invalid_path(valid_client): 191 | data = read_json_file("v2/responses/ls_invalid_path.json") 192 | with pytest.raises(UnexpectedStatusException) as excinfo: 193 | valid_client.list_files("cluster", "/invalid/path") 194 | 195 | byte_content = excinfo.value.responses[-1].content 196 | decoded_string = byte_content.decode('utf-8') 197 | response_dict = json.loads(decoded_string) 198 | message = response_dict["message"] 199 | 200 | assert str(message) == ( 201 | "ls: cannot access '/invalid/path': No such file or directory" 202 | ) 203 | 204 | 205 | def test_view(valid_client): 206 | data = read_json_file("v2/responses/view.json") 207 | resp = valid_client.view("cluster", "/home/user/file") 208 | 209 | assert resp == data["response"]["output"] 210 | 211 | 212 | def test_stat(valid_client): 213 | data = read_json_file("v2/responses/stat.json") 214 | resp = valid_client.stat("cluster", "/home/user/file") 215 | 216 | assert resp == data["response"]["output"] 217 | 218 | 219 | def test_stat_dereference(valid_client): 220 | data = read_json_file("v2/responses/stat_dereference.json") 221 | resp = valid_client.stat("cluster", "/home/user/file", 222 | dereference=True) 223 | 224 | assert resp == data["response"]["output"] 225 | 226 | 227 | def test_file_type(valid_client): 228 | data = read_json_file("v2/responses/file.json") 229 | resp = valid_client.file_type("cluster", "/home/user/file") 230 | 231 | assert resp == data["response"]["output"] 232 | 233 | 234 | def test_checksum(valid_client): 235 | data = read_json_file("v2/responses/checksum.json") 236 | resp = valid_client.checksum("cluster", "/home/user/file") 237 | 238 | assert resp == data["response"]["output"] 239 | 240 | 241 | def test_mkdir(valid_client): 242 | data = read_json_file("v2/responses/mkdir.json") 243 | resp = valid_client.mkdir("cluster", "/home/user/file") 244 | 245 | assert resp == data["response"]["output"] 246 | 247 | 248 | def test_chown(valid_client): 249 | data = read_json_file("v2/responses/chown.json") 250 | resp = valid_client.chown("cluster", "/home/user/file", 251 | "test1", "users") 252 | 253 | assert resp == data["response"]["output"] 254 | 255 | 256 | def test_chown_not_permitted(valid_client): 257 | data = read_json_file("v2/responses/chown_not_permitted.json") 258 | with pytest.raises(UnexpectedStatusException) as excinfo: 259 | valid_client.chown("cluster", "/home/test1/xxx", 260 | "test1", "users") 261 | 262 | assert str(excinfo.value) == ( 263 | f"last request: 403 {data['response']}: expected status 200" 264 | ) 265 | 266 | 267 | def test_chmod(valid_client): 268 | data = read_json_file("v2/responses/chmod.json") 269 | resp = valid_client.chmod("cluster", "/home/user/xxx", 270 | "777") 271 | 272 | assert resp == data["response"]["output"] 273 | 274 | 275 | def test_rm(valid_client): 276 | data = read_json_file("v2/responses/rm.json") 277 | resp = valid_client.rm("cluster", "/home/user/file") 278 | 279 | assert resp == data["response"]# ["output"] 280 | 281 | 282 | def test_job_info(valid_client): 283 | data = read_json_file("v2/responses/job_info.json") 284 | resp = valid_client.job_info("cluster") 285 | 286 | assert resp == data["response"]["jobs"] 287 | 288 | 289 | def test_job_info_jobid(valid_client): 290 | data = read_json_file("v2/responses/job_info.json") 291 | resp = valid_client.job_info("cluster", "1") 292 | 293 | assert resp == data["response"]["jobs"] 294 | 295 | 296 | def test_job_metadata(valid_client): 297 | data = read_json_file("v2/responses/job_metadata.json") 298 | resp = valid_client.job_metadata("cluster", "1") 299 | 300 | assert resp == data["response"]["jobs"] 301 | 302 | 303 | def test_job_submit(valid_client): 304 | data = read_json_file("v2/responses/job_submit.json") 305 | resp = valid_client.submit("cluster", "/path/to/dir", 306 | script_str="...") 307 | 308 | assert resp == data["response"] 309 | 310 | 311 | def test_job_submit_no_script(valid_client): 312 | with pytest.raises(ValueError) as excinfo: 313 | valid_client.submit("cluster", "/path/to/dir") 314 | 315 | assert str(excinfo.value) == ( 316 | "Exactly one of the arguments `script_str` or " 317 | "`script_local_path` must be set." 318 | ) 319 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | ; See https://tox.wiki 2 | 3 | [tox] 4 | envlist = py38 5 | 6 | [testenv] 7 | usedevelop = true 8 | 9 | [testenv:py{37,38,39,310,311}] 10 | extras = test 11 | commands = pytest {posargs} 12 | 13 | [testenv:docs] 14 | extras = docs 15 | commands = sphinx-build -nW --keep-going -b html docs/source docs/build/html 16 | -------------------------------------------------------------------------------- /utils/run_unasync.py: -------------------------------------------------------------------------------- 1 | import unasync 2 | 3 | 4 | unasync.unasync_files( 5 | ['firecrest/v2/_async/Client.py'], 6 | rules=[ 7 | unasync.Rule( 8 | fromdir="firecrest/v2/_async/", 9 | todir="firecrest/v2/_sync/", 10 | additional_replacements={ 11 | "AsyncFirecrest": "Firecrest", 12 | "AsyncClient": "Client", 13 | "aclose": "close", 14 | # "asyncio.sleep": "time.sleep", 15 | # multi token replacement doesn't work, it happens manually 16 | # TODO find a way to replace this automatically 17 | } 18 | ), 19 | ] 20 | ) 21 | --------------------------------------------------------------------------------