├── .coveragerc ├── .editorconfig ├── .github └── workflows │ ├── ci.yml │ ├── graphql-format.yml │ └── release.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── docs ├── CLI │ ├── admin │ │ ├── groups.md │ │ ├── images.md │ │ ├── instancetypes.md │ │ ├── reports.md │ │ ├── secrets.md │ │ ├── users.md │ │ └── volumes.md │ ├── apps.md │ ├── apptemplates.md │ ├── config.md │ ├── datasets.md │ ├── deployments.md │ ├── files.md │ ├── groups.md │ ├── images.md │ ├── info.md │ ├── instancetypes.md │ ├── jobs.md │ ├── me.md │ ├── models.md │ ├── notebooks.md │ ├── recurring-jobs.md │ ├── secrets.md │ └── volumes.md ├── configuration.md ├── design-notes.md ├── notebook │ ├── 00-getting-started.ipynb │ ├── admin │ │ ├── groups.ipynb │ │ ├── images.ipynb │ │ ├── instancetypes.ipynb │ │ ├── secrets.ipynb │ │ ├── users.ipynb │ │ └── volumes.ipynb │ ├── apps.ipynb │ ├── apptemplates.ipynb │ ├── config.ipynb │ ├── datasets.ipynb │ ├── deployments.ipynb │ ├── files.ipynb │ ├── groups.ipynb │ ├── images.ipynb │ ├── instancetypes.ipynb │ ├── jobs.ipynb │ ├── me.ipynb │ ├── models.ipynb │ ├── notebooks.ipynb │ ├── recurring-jobs.ipynb │ ├── secrets.ipynb │ └── volumes.ipynb └── screenplay │ ├── README.md │ ├── image-wise-operation.ipynb │ ├── instance-type-operation.ipynb │ ├── user-wise-operation.ipynb │ └── volume-wise-operation.ipynb ├── mypy.ini ├── primehub-sdk-autocomplete.sh ├── primehub ├── VERSION ├── __init__.py ├── admin_groups.py ├── admin_images.py ├── admin_instancetypes.py ├── admin_reports.py ├── admin_secrets.py ├── admin_users.py ├── admin_volumes.py ├── apps.py ├── apptemplates.py ├── cli.py ├── config.py ├── datasets.py ├── deployments.py ├── extras │ ├── __init__.py │ ├── devlab.py │ ├── doc_generator.py │ ├── e2e.py │ └── templates │ │ ├── cli.tpl.md │ │ └── examples │ │ ├── admin │ │ ├── groups.md │ │ ├── images.md │ │ ├── instancetypes.md │ │ ├── reports.md │ │ ├── secrets.md │ │ ├── users.md │ │ └── volumes.md │ │ ├── apps.md │ │ ├── apptemplates.md │ │ ├── config.md │ │ ├── datasets.md │ │ ├── deployments.md │ │ ├── files.md │ │ ├── groups.md │ │ ├── images.md │ │ ├── info.md │ │ ├── instancetypes.md │ │ ├── jobs.md │ │ ├── me.md │ │ ├── models.md │ │ ├── notebooks.md │ │ ├── recurring-jobs.md │ │ ├── secrets.md │ │ └── volumes.md ├── files.py ├── groups.py ├── images.py ├── info.py ├── instancetypes.py ├── jobs.py ├── me.py ├── models.py ├── notebooks.py ├── recurring_jobs.py ├── resource_operations.py ├── secrets.py ├── utils │ ├── __init__.py │ ├── argparser │ │ └── __init__.py │ ├── completion.py │ ├── core.py │ ├── decorators.py │ ├── display.py │ ├── http_client.py │ ├── optionals.py │ ├── permission.py │ └── validator.py ├── version.py └── volumes.py ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── graphql_formatter.py ├── test_admin_groups.py ├── test_admin_images.py ├── test_admin_instancetypes.py ├── test_admin_users.py ├── test_cli_config.py ├── test_cmd_config.py ├── test_cmd_me.py ├── test_config.py ├── test_datasets.py ├── test_files.py ├── test_graphql_lint.py ├── test_group_resource_operation.py ├── test_http_utils.py ├── test_implementation_in_source_level.py ├── test_jobs.py ├── test_output_utils.py ├── test_print.py ├── test_recurring_jobs.py ├── test_sdk_to_admin_cli.py ├── test_sdk_to_cli.py ├── test_sdk_to_cli_module_alias.py ├── test_test_utils.py ├── test_validator_tools.py └── test_volumes.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | venv/* 4 | primehub/extras/* 5 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [{Makefile,Makefile.*}] 12 | indent_style = tab 13 | 14 | [*.{py,rst}] 15 | indent_style = space 16 | indent_size = 4 17 | 18 | [*.{md,markdown}] 19 | trim_trailing_whitespace = false 20 | 21 | [*.json] 22 | indent_size = 2 23 | insert_final_newline = ignore 24 | 25 | [**.min.js] 26 | indent_style = ignore 27 | insert_final_newline = ignore -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: CI tests 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | branches: [ main ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install --upgrade pip 30 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 31 | - name: Run tests 32 | run: | 33 | make test 34 | -------------------------------------------------------------------------------- /.github/workflows/graphql-format.yml: -------------------------------------------------------------------------------- 1 | name: GraphQL format checker 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | lint: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: true 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: "3.8" 22 | - uses: actions/setup-node@v2 23 | with: 24 | node-version: '14' 25 | 26 | - name: Cache lint metadata 27 | uses: actions/cache@v2 28 | with: 29 | path: /tmp/graphql_lint 30 | key: graphql-lint-data 31 | 32 | - name: Install dependencies 33 | run: | 34 | python -m pip install --upgrade pip 35 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 36 | - name: Run graphql format checker 37 | run: | 38 | npm install -g prettier 39 | echo "go testing" 40 | make test-gql 41 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Release to PyPI 5 | 6 | on: 7 | push: 8 | tags: 'v*' 9 | 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | python-version: [ 3.8 ] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 29 | - name: Run tests 30 | run: | 31 | make test 32 | - name: Release 33 | run: | 34 | # get version from GITHUB_REF 35 | # input: "refs/tags/v0.1.0" 36 | # outpu: "0.1.0" 37 | 38 | # update version number 39 | echo "${GITHUB_REF:11}" > ./primehub/VERSION 40 | 41 | # generate pypirc 42 | echo "$PYPIRC" > $HOME/.pypirc 43 | 44 | # release to PyPI 45 | make release 46 | env: 47 | PYPIRC: ${{ secrets.PYPI }} 48 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include primehub *.py 2 | include primehub/*.json 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: test-html 2 | 3 | 4 | dev-requires: 5 | pip install -e .[dev] 6 | 7 | test: dev-requires 8 | py.test --cov=primehub --cov-report xml --flake8 --mypy --ignore=tests/test_graphql_lint.py 9 | 10 | test-html: dev-requires 11 | py.test --cov=primehub --cov-report html --flake8 --mypy --ignore=tests/test_graphql_lint.py 12 | 13 | test-regression: 14 | PRIMEHUB_SDK_DEVLAB=true primehub e2e basic-functions 15 | 16 | test-gql: dev-requires 17 | py.test tests/test_graphql_lint.py 18 | 19 | docs: dev-requires 20 | doc-primehub 21 | 22 | pre-release: dev-requires 23 | pip install build 24 | python3 -m build 25 | python3 -m twine upload --repository testpypi dist/* 26 | 27 | release: dev-requires 28 | pip install build 29 | python3 -m build 30 | python3 -m twine upload dist/* 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PrimeHub Python SDK 2 | 3 | PrimeHub Python SDK is the PrimeHub AI Platform Software Development Kit (SDK) for Python, which allows Python 4 | developers to write software that makes use of services like Job and Deployment. 5 | 6 | ## Getting Started 7 | 8 | Assuming that you have Python, you can install the library using pip: 9 | 10 | ``` 11 | $ pip install primehub-python-sdk 12 | ``` 13 | 14 | Or trying the latest from our source code: 15 | 16 | ``` 17 | $ pip install git+https://github.com/InfuseAI/primehub-python-sdk.git@main 18 | ``` 19 | 20 | ## Using CLI 21 | 22 | After installing PrimeHub Python SDK 23 | 24 | Next, set up the configuration in `~/.primehub/config.json`: 25 | 26 | ```json 27 | { 28 | "api-token": "", 29 | "endpoint": "https:///api/graphql", 30 | "group": { 31 | "name": "" 32 | } 33 | } 34 | ``` 35 | 36 | The `` could [be generated from User Portal](https://docs.primehub.io/docs/tasks/api-token). 37 | 38 | Then, from a shell: 39 | 40 | ``` 41 | $ primehub me 42 | id: a7db12dc-04fa-419c-9cd7-af768575a871 43 | username: phadmin 44 | firstName: None 45 | lastName: None 46 | email: dev+phadmin@infuseai.io 47 | isAdmin: True 48 | ``` 49 | 50 | Running `primehub` without arguments to show help: 51 | 52 | ``` 53 | $ primehub 54 | Usage: 55 | primehub 56 | 57 | Available Commands: 58 | admin Commands for system administrator 59 | apps Manage PrimeHub Applications 60 | apptemplates Get PhAppTemplates 61 | config Update the settings of PrimeHub SDK 62 | datasets Manage datasets 63 | deployments Get a deployment or list deployments 64 | files List and download shared files 65 | groups Get a group or list groups 66 | images Get a image or list images 67 | info Display the user information and the selected group information 68 | instancetypes Get an instance types of list instance types 69 | jobs Manage jobs 70 | me Show user account 71 | models Manage models 72 | notebooks Get notebooks logs 73 | recurring-jobs Manage recurring jobs 74 | version Display the version of PrimeHub Python SDK 75 | volumes Get a volume or list volumes 76 | 77 | Options: 78 | -h, --help Show the help 79 | 80 | Global Options: 81 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 82 | --endpoint ENDPOINT Override the GraphQL API endpoint 83 | --token TOKEN Override the API Token 84 | --group GROUP Override the current group 85 | --json Output the json format (output human-friendly format by default) 86 | ``` 87 | 88 | ## SDK 89 | 90 | from a Python interpreter: 91 | 92 | ``` 93 | In [1]: from primehub import PrimeHub, PrimeHubConfig 94 | 95 | In [2]: ph = PrimeHub(PrimeHubConfig()) 96 | 97 | In [3]: ph.me.me() 98 | Out[3]: 99 | {'id': 'a7db12dc-04fa-419c-9cd7-af768575a871', 100 | 'username': 'phadmin', 101 | 'firstName': None, 102 | 'lastName': None, 103 | 'email': 'dev+phadmin@infuseai.io', 104 | 'isAdmin': True} 105 | 106 | In [4]: 107 | ``` 108 | 109 | ## Docs 110 | 111 | There is a [docs](https://github.com/InfuseAI/primehub-python-sdk/tree/main/docs) folder in our repository. You could find: 112 | 113 | * [CLI](https://github.com/InfuseAI/primehub-python-sdk/tree/main/docs/CLI): all commands usage examples 114 | * [notebook](https://github.com/InfuseAI/primehub-python-sdk/tree/main/docs/notebook): examples written in python 115 | -------------------------------------------------------------------------------- /docs/CLI/admin/reports.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Reports 3 | 4 | ``` 5 | Usage: 6 | primehub admin reports 7 | 8 | Get reports 9 | 10 | Available Commands: 11 | download Download a report by url 12 | list List reports 13 | 14 | Options: 15 | -h, --help Show the help 16 | 17 | Global Options: 18 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 19 | --endpoint ENDPOINT Override the GraphQL API endpoint 20 | --token TOKEN Override the API Token 21 | --group GROUP Override the current group 22 | --json Output the json format (output human-friendly format by default) 23 | 24 | ``` 25 | 26 | 27 | ### download 28 | 29 | Download a report by url 30 | 31 | 32 | ``` 33 | primehub admin reports download 34 | ``` 35 | 36 | * url: The report url. 37 | 38 | 39 | * *(optional)* dest: The local path to save the report csv file 40 | 41 | 42 | 43 | 44 | ### list 45 | 46 | List reports 47 | 48 | 49 | ``` 50 | primehub admin reports list 51 | ``` 52 | 53 | 54 | * *(optional)* page: the page of all data 55 | 56 | 57 | 58 | 59 | 60 | ## Examples 61 | 62 | 63 | ### Reports list and download 64 | 65 | List reports 66 | 67 | ``` 68 | $ primehub admin reports list 69 | primehub admin reports list --json | jq . 70 | [ 71 | { 72 | "id": "2022/12", 73 | "summaryUrl": "https://primehub-python-sdk.primehub.io/api/report/monthly/2022/12", 74 | "detailedUrl": "https://primehub-python-sdk.primehub.io/api/report/monthly/details/2022/12" 75 | } 76 | ] 77 | ``` 78 | 79 | Download a report by url: 80 | 81 | ``` 82 | $ primehub admin reports download https://primehub-python-sdk.primehub.io/api/report/monthly/details/2022/12 83 | filename: 202212_details.csv 84 | ``` 85 | 86 | Download a report with `--dest` to change the download path: 87 | 88 | ``` 89 | $ primehub admin reports download https://primehub-python-sdk.primehub.io/api/report/monthly/2022/12 --dest foo/bar/202212.csv 90 | filename: /home/primehub/foo/bar/202212.csv 91 | ``` -------------------------------------------------------------------------------- /docs/CLI/apptemplates.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Apptemplates 3 | 4 | ``` 5 | Usage: 6 | primehub apptemplates 7 | 8 | Get PhAppTemplates 9 | 10 | Available Commands: 11 | get Get a PhApp template 12 | list List PhApp templates 13 | 14 | Options: 15 | -h, --help Show the help 16 | 17 | Global Options: 18 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 19 | --endpoint ENDPOINT Override the GraphQL API endpoint 20 | --token TOKEN Override the API Token 21 | --group GROUP Override the current group 22 | --json Output the json format (output human-friendly format by default) 23 | 24 | ``` 25 | 26 | 27 | ### get 28 | 29 | Get a PhApp template 30 | 31 | 32 | ``` 33 | primehub apptemplates get 34 | ``` 35 | 36 | * id 37 | 38 | 39 | 40 | 41 | 42 | ### list 43 | 44 | List PhApp templates 45 | 46 | 47 | ``` 48 | primehub apptemplates list 49 | ``` 50 | 51 | 52 | 53 | 54 | 55 | 56 | ## Examples 57 | 58 | The `list` action will show you each template that is registered to the PrimeHub 59 | 60 | ``` 61 | primehub apptemplates list 62 | ``` 63 | 64 | ``` 65 | id: code-server 66 | name: Code Server 67 | version: v3.9.2 68 | description: Run VS Code on any machine anywhere and access it in the browser. 69 | docLink: https://github.com/cdr/code-server 70 | image: codercom/code-server:3.9.2 71 | 72 | id: label-studio 73 | name: Label Studio 74 | version: 1.1.0 75 | description: Label Studio is an open source data labeling tool for labeling and exploring multiple types of data. You can perform many different types of labeling for many different data formats. 76 | docLink: https://labelstud.io/guide/ 77 | image: heartexlabs/label-studio:1.1.0 78 | 79 | id: matlab 80 | name: Matlab 81 | version: r2020b 82 | description: MATLAB is a programming platform designed for engineers and scientists. The MATLAB Deep Learning Container provides algorithms, pretrained models, and apps to create, train, visualize, and optimize deep neural networks. 83 | docLink: https://ngc.nvidia.com/catalog/containers/partners:matlab/tags 84 | image: nvcr.io/partners/matlab:r2020b 85 | 86 | id: mlflow 87 | name: MLflow 88 | version: v1.9.1 89 | description: MLflow is an open source platform to manage the ML lifecycle, including experimentation, reproducibility, deployment, and a central model registry. 90 | docLink: https://www.mlflow.org/docs/1.9.1/index.html 91 | image: larribas/mlflow:1.9.1 92 | 93 | id: streamlit 94 | name: Streamlit 95 | version: v0.79.0 96 | description: Streamlit turns data scripts into shareable web apps in minutes. All in Python. All for free. No front‑end experience required. 97 | docLink: https://docs.primehub.io/docs/primehub-app-builtin-streamlit 98 | image: infuseai/streamlit:v0.79.0 99 | ``` 100 | 101 | You could get one of them with `get` action and the id of a template: 102 | 103 | ``` 104 | primehub apptemplates get code-server 105 | ``` 106 | 107 | 108 | ``` 109 | id: code-server 110 | name: Code Server 111 | version: v3.9.2 112 | description: Run VS Code on any machine anywhere and access it in the browser. 113 | docLink: https://github.com/cdr/code-server 114 | image: codercom/code-server:3.9.2 115 | ``` -------------------------------------------------------------------------------- /docs/CLI/config.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Config 3 | 4 | ``` 5 | Usage: 6 | primehub config 7 | 8 | Update the settings of PrimeHub SDK 9 | 10 | Available Commands: 11 | generate-token Generate Token Flow 12 | set-endpoint Set endpoint and save to the config file 13 | set-group Set group and save to the config file 14 | set-token Set token and save to the config file 15 | 16 | Options: 17 | -h, --help Show the help 18 | 19 | Global Options: 20 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 21 | --endpoint ENDPOINT Override the GraphQL API endpoint 22 | --token TOKEN Override the API Token 23 | --group GROUP Override the current group 24 | --json Output the json format (output human-friendly format by default) 25 | 26 | ``` 27 | 28 | 29 | ### generate-token 30 | 31 | Generate Token Flow 32 | 33 | 34 | ``` 35 | primehub config generate-token 36 | ``` 37 | 38 | * server_url: PrimeHub's URL 39 | 40 | 41 | 42 | 43 | 44 | ### set-endpoint 45 | 46 | Set endpoint and save to the config file 47 | 48 | 49 | ``` 50 | primehub config set-endpoint 51 | ``` 52 | 53 | * endpoint: an URL to GraphQL API endpoint 54 | 55 | 56 | 57 | 58 | 59 | ### set-group 60 | 61 | Set group and save to the config file 62 | 63 | 64 | ``` 65 | primehub config set-group 66 | ``` 67 | 68 | * group: group name 69 | 70 | 71 | 72 | 73 | 74 | ### set-token 75 | 76 | Set token and save to the config file 77 | 78 | 79 | ``` 80 | primehub config set-token 81 | ``` 82 | 83 | * token: a token used by GraphQL request 84 | 85 | 86 | 87 | 88 | 89 | 90 | ## Examples 91 | 92 | The `config` command provides the ability to update the PrimeHub SDK configuration file. The file locates 93 | at `~/.primehub/config.json` by default. 94 | 95 | > Be careful with the config command, it will `MODIFY` the configuration file directly 96 | 97 | ### Example: create a config file 98 | 99 | We could use the `config` command with the `--config` flag to make a new config to the `my.config` path. 100 | 101 | ``` 102 | primehub config set-endpoint http://primehub-python-sdk.primehub.io --config my-config.json 103 | primehub config set-token my-token --config my-config.json 104 | primehub config set-group no-such-group --config my-config.json 105 | ``` 106 | 107 | You might see the warning, because of the fake endpoint: 108 | 109 | ``` 110 | 2021-08-03 11:05:17,209 - cmd-config - WARNING - Got request problem with http://primehub-python-sdk.primehub.io: HTTPConnectionPool(host='primehub-python-sdk.primehub.io', port=80): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')) 111 | No group information for [no-such-group], please set the right group 112 | ``` 113 | 114 | However, `config` still create the file for you: 115 | 116 | ```json 117 | { 118 | "endpoint": "http://primehub-python-sdk.primehub.io", 119 | "api-token": "my-token", 120 | "group": { 121 | "name": null 122 | } 123 | } 124 | ``` 125 | 126 | The SDK can not fetch the group information from the server, finally you got a group with null name. If you did 127 | configure with a real api server, the group information would be available: 128 | 129 | ```json 130 | { 131 | "endpoint": "http://primehub-python-sdk.primehub.io", 132 | "api-token": "my-token", 133 | "group": { 134 | "id": "927406f6-88b0-490e-8e36-7835224fdf13", 135 | "name": "sdk", 136 | "displayName": "PrimeHub Python SDK Team" 137 | } 138 | } 139 | ``` 140 | 141 | ### Example: generate API Token with OAuth login 142 | 143 | ``` 144 | primehub config generate-token http://primehub-python-sdk.primehub.io 145 | ``` 146 | 147 | The commands will guide you how to get an API token, it will show 2 required actions: 148 | * visit the URL and login with your account and copy the authorization code from the web page 149 | * paste the authorization code back to the terminal 150 | 151 | 152 | ``` 153 | Go to this URL in the browser http://primehub-python-sdk.primehub.io/console/oidc/auth-flow/request 154 | Enter your authorization code: 155 | ``` 156 | 157 | When all actions have done, the configuration got updated: 158 | 159 | ``` 160 | Found old configuration, backup it to /home/phadmin/.primehub/config-20211001161504.json 161 | PrimeHub SDK Config has been updated: /home/phadmin/.primehub/config.json 162 | ``` -------------------------------------------------------------------------------- /docs/CLI/files.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Files 3 | 4 | ``` 5 | Usage: 6 | primehub files 7 | 8 | List and download shared files 9 | 10 | Available Commands: 11 | delete delete shared files 12 | download Download shared files 13 | get-phfs-uri Get PHFS URI 14 | list List shared files 15 | upload Upload shared files 16 | 17 | Options: 18 | -h, --help Show the help 19 | 20 | Global Options: 21 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 22 | --endpoint ENDPOINT Override the GraphQL API endpoint 23 | --token TOKEN Override the API Token 24 | --group GROUP Override the current group 25 | --json Output the json format (output human-friendly format by default) 26 | 27 | ``` 28 | 29 | 30 | ### delete 31 | 32 | delete shared files 33 | 34 | 35 | ``` 36 | primehub files delete 37 | ``` 38 | 39 | * path: The path of file or folder 40 | 41 | 42 | * *(optional)* recursive: Delete recursively, it works when a path is a directory. 43 | 44 | 45 | 46 | 47 | ### download 48 | 49 | Download shared files 50 | 51 | 52 | ``` 53 | primehub files download 54 | ``` 55 | 56 | * path: The path of file or folder 57 | * dest: The local path to save artifacts 58 | 59 | 60 | * *(optional)* recursive 61 | 62 | 63 | 64 | 65 | ### get-phfs-uri 66 | 67 | Get PHFS URI 68 | 69 | 70 | ``` 71 | primehub files get-phfs-uri 72 | ``` 73 | 74 | * path: The path 75 | 76 | 77 | 78 | 79 | 80 | ### list 81 | 82 | List shared files 83 | 84 | 85 | ``` 86 | primehub files list 87 | ``` 88 | 89 | * path: The path to list 90 | 91 | 92 | 93 | 94 | 95 | ### upload 96 | 97 | Upload shared files 98 | 99 | 100 | ``` 101 | primehub files upload 102 | ``` 103 | 104 | * src: The local path to save artifacts 105 | * path: The path of file or folder 106 | 107 | 108 | * *(optional)* recursive 109 | 110 | 111 | 112 | 113 | 114 | ## Examples 115 | 116 | We could use `list` to watch files or directory remotely. 117 | 118 | `list` requires a `path` parameter, it always starts with `/`: 119 | 120 | ``` 121 | $ primehub files list / 122 | name size lastModified phfsUri 123 | ------------- ------ -------------- --------------------- 124 | jobArtifacts/ 0 phfs:///jobArtifacts/ 125 | ``` 126 | 127 | You might go deeply into a sub-directory: 128 | 129 | ``` 130 | $ primehub files list /jobArtifacts/job-202107290838-aoq173/ 131 | name size lastModified phfsUri 132 | ---------- ------ -------------- ------------------ 133 | .metadata/ 0 phfs:///.metadata/ 134 | ``` 135 | 136 | Then `download` a file directly or a directory with `--recursive` options 137 | 138 | ``` 139 | $ primehub files download /jobArtifacts/job-202107290838-aoq173 ./my-download --recursive 140 | ``` 141 | 142 | ``` 143 | ./my-download 144 | └── job-202107290838-aoq173 145 | └── .metadata 146 | 147 | 1 directory, 1 file 148 | ``` 149 | 150 | Uses `upload` to upload a file or a directory with `--recursive` options. 151 | 152 | *Note: it follows shell `cp -R` manner when upload a directory* 153 | > If the source_file ends in a /, the contents of the directory are copied rather than the directory itself. 154 | 155 | ``` 156 | $ primehub files upload ./my-download /jobArtifacts/job-202107290838-aoq173 --recursive 157 | [Uploading] ./my-download/job-202107290838-aoq173/.metadata -> phfs:///jobArtifacts/job-202107290838-aoq173/my-download/job-202107290838-aoq173/.metadata 158 | success phfs file 159 | --------- ----------------------------------------------------------------------------------- ----------------------------------------------- 160 | True /jobArtifacts/job-202107290838-aoq173/my-download/job-202107290838-aoq173/.metadata ./my-download/job-202107290838-aoq173/.metadata 161 | ``` 162 | 163 | ``` 164 | $ primehub files upload ./my-download/ /jobArtifacts/job-202107290838-aoq173 --recursive 165 | [Uploading] ./my-download/job-202107290838-aoq173/.metadata -> phfs:///jobArtifacts/job-202107290838-aoq173/job-202107290838-aoq173/.metadata 166 | success phfs file 167 | --------- ----------------------------------------------------------------------- ----------------------------------------------- 168 | True /jobArtifacts/job-202107290838-aoq173/job-202107290838-aoq173/.metadata ./my-download/job-202107290838-aoq173/.metadata 169 | ``` 170 | 171 | Uses `delete` to delete a file or a directory with `--recursive` options. 172 | 173 | ``` 174 | $ primehub files delete /jobArtifacts/job-202107290838-aoq173 --recursive 175 | deleteFiles: 3 176 | ``` -------------------------------------------------------------------------------- /docs/CLI/images.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Images 3 | 4 | ``` 5 | Usage: 6 | primehub images 7 | 8 | Get a image or list images 9 | 10 | Available Commands: 11 | create Create an image 12 | delete Delete an image by name 13 | get Get an image by name 14 | list List images 15 | 16 | Options: 17 | -h, --help Show the help 18 | 19 | Global Options: 20 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 21 | --endpoint ENDPOINT Override the GraphQL API endpoint 22 | --token TOKEN Override the API Token 23 | --group GROUP Override the current group 24 | --json Output the json format (output human-friendly format by default) 25 | 26 | ``` 27 | 28 | 29 | ### create 30 | 31 | Create an image 32 | 33 | 34 | ``` 35 | primehub images create 36 | ``` 37 | 38 | 39 | * *(optional)* file: The file path of the configurations 40 | 41 | 42 | 43 | 44 | ### delete 45 | 46 | Delete an image by name 47 | 48 | 49 | ``` 50 | primehub images delete 51 | ``` 52 | 53 | * name 54 | 55 | 56 | 57 | 58 | 59 | ### get 60 | 61 | Get an image by name 62 | 63 | 64 | ``` 65 | primehub images get 66 | ``` 67 | 68 | * name: the name of an image 69 | 70 | 71 | 72 | 73 | 74 | ### list 75 | 76 | List images 77 | 78 | 79 | ``` 80 | primehub images list 81 | ``` 82 | 83 | 84 | 85 | 86 | 87 | 88 | ## Examples 89 | 90 | The `images` command is a group specific resource. It only works after the `group` assigned. 91 | 92 | Using `list` to find all images in your group: 93 | 94 | ``` 95 | primehub images list 96 | ``` 97 | 98 | ```name displayName description 99 | ------------- ------------------------------ ------------------------------ 100 | pytorch-1 PyTorch 1.8.0 (Python 3.7) PyTorch 1.8.0 (Python 3.7) 101 | tf-2 TensorFlow 2.5.0 (Python 3.7) TensorFlow 2.5.0 (Python 3.7) 102 | base-notebook base-notebook base notebook 103 | tf-1 TensorFlow 1.15.4 (Python 3.7) TensorFlow 1.15.4 (Python 3.7 104 | ``` 105 | 106 | If you already know the name of a images, use the `get` to get a single entry: 107 | 108 | ``` 109 | primehub images get tf-2 110 | ``` 111 | 112 | ``` 113 | id: tf-2 114 | name: tf-2 115 | displayName: TensorFlow 2.5.0 (Python 3.7) 116 | description: TensorFlow 2.5.0 (Python 3.7) 117 | useImagePullSecret: None 118 | spec: 119 | description: TensorFlow 2.5.0 (Python 3.7) 120 | displayName: TensorFlow 2.5.0 (Python 3.7) 121 | type: both 122 | url: infuseai/docker-stacks:tensorflow-notebook-v2-5-0-63fdf50a 123 | urlForGpu: infuseai/docker-stacks:tensorflow-notebook-v2-5-0-63fdf50a-gpu-cuda-11 124 | ``` 125 | 126 | Create a group image: 127 | 128 | ``` 129 | $ primehub images create < 7 | 8 | Display the user information and the selected group information 9 | 10 | Available Commands: 11 | info Show PrimeHub Cli information 12 | 13 | Options: 14 | -h, --help Show the help 15 | 16 | Global Options: 17 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 18 | --endpoint ENDPOINT Override the GraphQL API endpoint 19 | --token TOKEN Override the API Token 20 | --group GROUP Override the current group 21 | --json Output the json format (output human-friendly format by default) 22 | 23 | ``` 24 | 25 | 26 | ### info 27 | 28 | Show PrimeHub Cli information 29 | 30 | 31 | ``` 32 | primehub info info 33 | ``` 34 | 35 | 36 | 37 | 38 | 39 | 40 | ## Examples 41 | 42 | `info` command is only for `primehub` command line. It shows a human-readable output that gives your an outline under 43 | the account: 44 | 45 | ``` 46 | primehub info 47 | ``` 48 | 49 | ``` 50 | Endpoint: https://example.primehub.io/api/graphql 51 | User: 52 | Id: a7db12dc-04fa-419c-9cd7-af768575a871 53 | Username: phadmin 54 | Email: dev+phadmin@infuseai.io 55 | First Name: None 56 | Last Name: None 57 | Is Admin: True 58 | Current Group: 59 | Id: 2b080113-e2f1-4b1b-a6ef-eb0ca5e2f376 60 | Name: phusers 61 | Display Name: primehub users 62 | Group Quota: 63 | CPU: None 64 | GPU: 0 65 | Memory: None 66 | User Quota: 67 | CPU: None 68 | GPU: 0 69 | Memory: None 70 | Images: 71 | pytorch-1 72 | tf-2 73 | base-notebook 74 | tf-1 75 | InstanceTypes: 76 | cpu-1 77 | gpu-2 78 | cpu-2 79 | gpu-1 80 | Volumes: 81 | ``` 82 | 83 | ### Notes 84 | 85 | The two commands are the same, because the `info` command group only has one command `info`, it could use shortcut form: 86 | 87 | ``` 88 | # shortcut form 89 | primehub info 90 | ``` 91 | 92 | ``` 93 | # 94 | primehub info info 95 | ``` 96 | -------------------------------------------------------------------------------- /docs/CLI/instancetypes.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Instancetypes 3 | 4 | ``` 5 | Usage: 6 | primehub instancetypes 7 | 8 | Get an instance types of list instance types 9 | 10 | Available Commands: 11 | get Get an instance type by name 12 | list List instance types 13 | 14 | Options: 15 | -h, --help Show the help 16 | 17 | Global Options: 18 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 19 | --endpoint ENDPOINT Override the GraphQL API endpoint 20 | --token TOKEN Override the API Token 21 | --group GROUP Override the current group 22 | --json Output the json format (output human-friendly format by default) 23 | 24 | ``` 25 | 26 | 27 | ### get 28 | 29 | Get an instance type by name 30 | 31 | 32 | ``` 33 | primehub instancetypes get 34 | ``` 35 | 36 | * name: the name of an instance type 37 | 38 | 39 | 40 | 41 | 42 | ### list 43 | 44 | List instance types 45 | 46 | 47 | ``` 48 | primehub instancetypes list 49 | ``` 50 | 51 | 52 | 53 | 54 | 55 | 56 | ## Examples 57 | 58 | The `instancetypes` command is a group specific resource. It only works after the `group` assigned. 59 | 60 | Using `list` to find all instance types in your group: 61 | 62 | ``` 63 | primehub instancetypes list 64 | ``` 65 | 66 | ``` 67 | name displayName description cpuRequest cpuLimit memoryRequest memoryLimit gpuLimit global 68 | -------------------------------- --------------------------------------------- ------------------------------ ------------ ---------- --------------- ------------- ---------- -------- 69 | cpu-2 CPU 2 2 CPU / 10G 2 10 0 True 70 | gpu-1 GPU 1 2 CPU / 7G / 1 GPU 1.5 2 5 7 1 True 71 | gpu-2 GPU 2 4 CPU / 14G / 2 GPU 4 4 14 14 2 True 72 | cpu-1 CPU 1 1 CPU / 2G 1 1 2 2 0 True 73 | ``` 74 | 75 | If you already know the name of an instance type, use the `get` to get a single entry: 76 | 77 | ``` 78 | primehub instancetypes get cpu-1 79 | ``` 80 | 81 | ``` 82 | name: cpu-1 83 | displayName: CPU 1 84 | description: 1 CPU / 2G 85 | cpuRequest: 1 86 | cpuLimit: 1 87 | memoryRequest: 2 88 | memoryLimit: 2 89 | gpuLimit: 0 90 | global: True 91 | tolerations: [{'operator': 'Equal', 'key': 'hub.jupyter.org/dedicated', 'value': 'user', 'effect': 'NoSchedule'}] 92 | nodeSelector: None 93 | ``` -------------------------------------------------------------------------------- /docs/CLI/me.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Me 3 | 4 | ``` 5 | Usage: 6 | primehub me 7 | 8 | Show user account 9 | 10 | Available Commands: 11 | me Get user information 12 | 13 | Options: 14 | -h, --help Show the help 15 | 16 | Global Options: 17 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 18 | --endpoint ENDPOINT Override the GraphQL API endpoint 19 | --token TOKEN Override the API Token 20 | --group GROUP Override the current group 21 | --json Output the json format (output human-friendly format by default) 22 | 23 | ``` 24 | 25 | 26 | ### me 27 | 28 | Get user information 29 | 30 | 31 | ``` 32 | primehub me me 33 | ``` 34 | 35 | 36 | 37 | 38 | 39 | 40 | ## Examples 41 | 42 | `me` command shows the account information. It is useful when you have no idea with your api-token belongs to which 43 | user. 44 | 45 | ``` 46 | $ primehub me 47 | id: a7db12dc-04fa-419c-9cd7-af768575a871 48 | username: phadmin 49 | firstName: None 50 | lastName: None 51 | email: dev+phadmin@infuseai.io 52 | isAdmin: True 53 | ``` 54 | 55 | ### Notes 56 | 57 | The two commands are the same, because the `me` command group only has one command `me`, it could use shortcut form: 58 | 59 | ``` 60 | # shortcut form 61 | primehub me 62 | ``` 63 | 64 | ``` 65 | # 66 | primehub me me 67 | ``` 68 | -------------------------------------------------------------------------------- /docs/CLI/notebooks.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Notebooks 3 | 4 | ``` 5 | Usage: 6 | primehub notebooks 7 | 8 | Get notebooks logs 9 | 10 | Available Commands: 11 | logs Get notebooks logs 12 | 13 | Options: 14 | -h, --help Show the help 15 | 16 | Global Options: 17 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 18 | --endpoint ENDPOINT Override the GraphQL API endpoint 19 | --token TOKEN Override the API Token 20 | --group GROUP Override the current group 21 | --json Output the json format (output human-friendly format by default) 22 | 23 | ``` 24 | 25 | 26 | ### logs 27 | 28 | Get notebooks logs 29 | 30 | 31 | ``` 32 | primehub notebooks logs 33 | ``` 34 | 35 | 36 | * *(optional)* follow: Wait for additional logs to be appended 37 | 38 | * *(optional)* tail: Show last n lines 39 | 40 | 41 | 42 | 43 | 44 | ## Examples 45 | 46 | See log messages from the user's notebook 47 | 48 | ``` 49 | primehub notebooks logs 50 | ``` 51 | 52 | ``` 53 | Set username to: jovyan 54 | usermod: no changes 55 | Changing ownership of /datasets/kaggle to 1000:100 with options '' 56 | Changing ownership of /phfs to 1000:100 with options '' 57 | Changing ownership of /home/jovyan to 1000:100 with options '' 58 | Granting jovyan sudo access and appending /opt/conda/bin to sudo PATH 59 | Executing the command: jupyter labhub --ip=0.0.0.0 --port=8888 --NotebookApp.default_url=/lab 60 | [W 2021-08-03 10:01:20.065 SingleUserLabApp configurable:190] Config option `open_browser` not recognized by `SingleUserLabApp`. Did you mean one of: `browser, expose_app_in_browser`? 61 | [WARNING 2021-08-03 10:01:21.567 SingleUserLabApp zmqhandlers:275] Couldn't authenticate WebSocket connection 62 | [WARNING 2021-08-03 10:01:21.601 SingleUserLabApp log:181] 403 GET /user/phadmin/api/kernels/cc295b85-35dc-4c6b-b8fd-ac3885224738/channels?session_id=be318bb4-b241-4b57-83a3-63f10b746ba8 (@36.225.16.122) 35.68ms 63 | ``` 64 | 65 | ### Notes 66 | 67 | If the notebook has not been ready, you might get status message: 68 | 69 | ``` 70 | {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"container \"notebook\" in pod \"jupyter-phadmin\" is waiting to start: PodInitializing","reason":"BadRequest","code":400} 71 | ``` -------------------------------------------------------------------------------- /docs/CLI/secrets.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Secrets 3 | 4 | ``` 5 | Usage: 6 | primehub secrets 7 | 8 | Get a secret or list secrets 9 | 10 | Available Commands: 11 | get Get a secret by id 12 | list List secrets 13 | 14 | Options: 15 | -h, --help Show the help 16 | 17 | Global Options: 18 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 19 | --endpoint ENDPOINT Override the GraphQL API endpoint 20 | --token TOKEN Override the API Token 21 | --group GROUP Override the current group 22 | --json Output the json format (output human-friendly format by default) 23 | 24 | ``` 25 | 26 | 27 | ### get 28 | 29 | Get a secret by id 30 | 31 | 32 | ``` 33 | primehub secrets get 34 | ``` 35 | 36 | * id: the id of a secret 37 | 38 | 39 | 40 | 41 | 42 | ### list 43 | 44 | List secrets 45 | 46 | 47 | ``` 48 | primehub secrets list 49 | ``` 50 | 51 | 52 | 53 | 54 | 55 | 56 | ## Examples 57 | 58 | ### Query secrets 59 | 60 | `secrets` command is used for querying secrets. You will use `id` with features that support **image pull secret**, such as the `Deployments`. 61 | 62 | Find all secrets with `list` command: 63 | 64 | ``` 65 | $ primehub secrets list 66 | 67 | id name type 68 | -------------- -------- ---------- 69 | image-example1 example1 kubernetes 70 | ``` 71 | 72 | Get a secret with `get` command: 73 | 74 | ``` 75 | $ primehub secrets get image-example1 76 | 77 | id: image-example1 78 | name: example1 79 | type: kubernetes 80 | ``` -------------------------------------------------------------------------------- /docs/CLI/volumes.md: -------------------------------------------------------------------------------- 1 | 2 | # Primehub Volumes 3 | 4 | ``` 5 | Usage: 6 | primehub volumes 7 | 8 | Get a volume or list volumes 9 | 10 | Available Commands: 11 | get Get a volume by name 12 | list List volumes 13 | 14 | Options: 15 | -h, --help Show the help 16 | 17 | Global Options: 18 | --config CONFIG Change the path of the config file (Default: ~/.primehub/config.json) 19 | --endpoint ENDPOINT Override the GraphQL API endpoint 20 | --token TOKEN Override the API Token 21 | --group GROUP Override the current group 22 | --json Output the json format (output human-friendly format by default) 23 | 24 | ``` 25 | 26 | 27 | ### get 28 | 29 | Get a volume by name 30 | 31 | 32 | ``` 33 | primehub volumes get 34 | ``` 35 | 36 | * name: the name of a volume 37 | 38 | 39 | 40 | 41 | 42 | ### list 43 | 44 | List volumes 45 | 46 | 47 | ``` 48 | primehub volumes list 49 | ``` 50 | 51 | 52 | 53 | 54 | 55 | 56 | ## Examples 57 | 58 | The `volumes` command is a group specific resource. It only works after the `group` assigned. 59 | 60 | Using `list` to find all volumes in your group: 61 | 62 | ``` 63 | $ primehub volumes list 64 | ``` 65 | 66 | ``` 67 | id name displayName description type 68 | ------ ------ ------------- ------------- ------ 69 | kaggle kaggle kaggle pv 70 | ``` 71 | 72 | If you already know the name of a volume, use the `get` to get a single entry: 73 | 74 | ``` 75 | $ primehub volumes get kaggle 76 | ``` 77 | 78 | ``` 79 | primehub volumes get kaggle 80 | id: kaggle 81 | name: kaggle 82 | displayName: kaggle 83 | description: 84 | type: pv 85 | ``` -------------------------------------------------------------------------------- /docs/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | PrimeHub Python SDK provides the `primehub.PrimeHubConfig` class to set up itself. It works in this way: 4 | 5 | 1. load the default config path `~/.primehub/config.json` or load a specific path from the `config` argument 6 | 2. take settings from environment variables 7 | 3. overide from property setter 8 | 9 | ## Configuration file 10 | 11 | The configuration file looks like: 12 | 13 | ```json 14 | { 15 | "endpoint": "", 16 | "api-token": "", 17 | "group": { 18 | "id": "", 19 | "name": "", 20 | "displayName": "", 21 | } 22 | } 23 | ``` 24 | 25 | * api-token and endpoint: they were generated from the PrimeHub Console 26 | * group: the name of a group which is the active group to query with 27 | 28 | ## Environment Variables 29 | 30 | There are three environment variables, they could be mapped to the field in the configuration file: 31 | 32 | * PRIMEHUB_API_TOKEN maps to `api-token` 33 | * PRIMEHUB_API_ENDPOINT maps to `endpoint` 34 | * PRIMEHUB_GROUP maps to `group` 35 | 36 | If a environment exists and not a blank value, it will override the value from the configuration file. 37 | 38 | ## Property Setter 39 | 40 | We also provide a property setter to override one of the {endpoint, api-token, group}. They are available both of CLI and SDK: 41 | 42 | ```python 43 | cfg = PrimeHubConfig() 44 | cfg.group = 'set-a-different-group' 45 | ``` 46 | 47 | ```bash 48 | primehub --group set-a-different-group [command] ... 49 | ``` 50 | 51 | ## Evaluation Order 52 | 53 | We load variables in this order: 54 | 55 | 1. configuration file 56 | 2. environment variables 57 | 3. property 58 | 59 | The configuration will override by environment variables and environment variables will override by property. -------------------------------------------------------------------------------- /docs/notebook/00-getting-started.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Getting Started with PrimeHub Python SDK\n", 8 | "\n", 9 | "PrimeHub Python SDK makes you automation with PrimeHub Platform.\n" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "In order to make the SDK working, you have to \n", 17 | "* install the library with `pip`\n", 18 | "* create a config file in the `~/.primehub/config.json`" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": {}, 24 | "source": [ 25 | "## Install with pip\n", 26 | "\n", 27 | "* Let's install PrimeHub Python SDK with `pip`. " 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "!pip install primehub-python-sdk" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "## Request the API Token\n", 44 | "\n", 45 | "In order to get the token, you have to have an account in the PrimeHub cluster, the following process will ask you loing with your account.\n", 46 | "\n", 47 | "PLEASE UPDATE `PRIMEHUB_CLUSTER` to your cluster" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "PRIMEHUB_CLUSTER = 'http://primehub-python-sdk.primehub.io'\n", 57 | "\n", 58 | "from primehub import PrimeHub, PrimeHubConfig\n", 59 | "ph = PrimeHub(PrimeHubConfig())\n", 60 | "ph.config.generate(PRIMEHUB_CLUSTER)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "### Verify configration with PrimeHub Python SDK" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "if ph.is_ready():\n", 77 | " print(\"PrimeHub Python SDK setup successfully\")\n", 78 | " print(\"Current Group:\", ph.primehub_config.current_group)\n", 79 | "else:\n", 80 | " print(\"PrimeHub Python SDK couldn't get the group information, please check the configuration.\")" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "## Here is all available commands in the PrimeHub SDK\n", 88 | "\n", 89 | "We will talk each command in the individual notebook. Commands call convention are:\n", 90 | "\n", 91 | "`PrimeHub..(*args, **kwargs)`\n", 92 | "\n", 93 | "You could look up all supported commands with `get_all_commands()` method." 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "for cmd in ph.get_all_commands():\n", 103 | " print(cmd)" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": {}, 109 | "source": [ 110 | "## Please check command details \n", 111 | "\n", 112 | "* Configuration and account resources\n", 113 | " * config.ipynb\n", 114 | " * groups.ipynb\n", 115 | " * me.ipynb\n", 116 | " \n", 117 | "* Group resource(group-resources.ipynb):\n", 118 | " * datasets\n", 119 | " * images\n", 120 | " * instancetypes\n", 121 | "\n", 122 | "* Job related\n", 123 | " * files\n", 124 | " * jobs\n", 125 | " * schedules\n", 126 | "\n", 127 | "* notebooks command\n" 128 | ] 129 | } 130 | ], 131 | "metadata": { 132 | "interpreter": { 133 | "hash": "b2ec084f4af1cfdda5df94d11be181b9273da0e6177ccab952b78bb7bed491ac" 134 | }, 135 | "kernelspec": { 136 | "display_name": "PrimeHub SDK", 137 | "language": "python", 138 | "name": "myenv" 139 | }, 140 | "language_info": { 141 | "codemirror_mode": { 142 | "name": "ipython", 143 | "version": 3 144 | }, 145 | "file_extension": ".py", 146 | "mimetype": "text/x-python", 147 | "name": "python", 148 | "nbconvert_exporter": "python", 149 | "pygments_lexer": "ipython3", 150 | "version": "3.7.6" 151 | } 152 | }, 153 | "nbformat": 4, 154 | "nbformat_minor": 4 155 | } 156 | -------------------------------------------------------------------------------- /docs/notebook/apptemplates.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Apptemplates command\n", 8 | "\n", 9 | "### Introduction\n", 10 | "\n", 11 | "The `apptemplates` command make you get information about PhApplication templates\n", 12 | "\n", 13 | "* list: list all PhApp templates\n", 14 | "* get: get a template by id" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "## Setup PrimeHub Python SDK\n" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from primehub import PrimeHub, PrimeHubConfig\n", 31 | "ph = PrimeHub(PrimeHubConfig())\n", 32 | "\n", 33 | "if ph.is_ready():\n", 34 | " print(\"PrimeHub Python SDK setup successfully\")\n", 35 | "else:\n", 36 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "## Help documentation" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "help(ph.apptemplates)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "## Examples" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "### List PhApp templates" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "# List all jobs or with page number\n", 76 | "for template in ph.apptemplates.list():\n", 77 | " print(f'id: {template[\"id\"]}\\ndescription: {template[\"description\"]}\\n')\n" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "### Get a template by id" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "ph.apptemplates.get('code-server')" 94 | ] 95 | } 96 | ], 97 | "metadata": { 98 | "kernelspec": { 99 | "display_name": "PrimeHub SDK", 100 | "language": "python", 101 | "name": "myenv" 102 | }, 103 | "language_info": { 104 | "codemirror_mode": { 105 | "name": "ipython", 106 | "version": 3 107 | }, 108 | "file_extension": ".py", 109 | "mimetype": "text/x-python", 110 | "name": "python", 111 | "nbconvert_exporter": "python", 112 | "pygments_lexer": "ipython3", 113 | "version": "3.7.6" 114 | } 115 | }, 116 | "nbformat": 4, 117 | "nbformat_minor": 5 118 | } 119 | -------------------------------------------------------------------------------- /docs/notebook/config.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1dbeaa6f", 6 | "metadata": {}, 7 | "source": [ 8 | "# Config command\n", 9 | "\n", 10 | "### Introduction\n", 11 | "\n", 12 | "Config command can setup `PrimeHubConfig` or overwrite the configuration file (`~/.primehub/config.json`).\n", 13 | "\n", 14 | "* set_endpoint: the endpoint of PrimeHub GraphQL API. \n", 15 | "* set_token: API token, you could switch users by changing API Token.\n", 16 | "* set_group: choose the current group.\n", 17 | "\n", 18 | "\n", 19 | "### Note: current group\n", 20 | "\n", 21 | "Some reousrces are group related, it requires a `group` to checking or filtering the results. \n", 22 | "When `set_group` called, `config` will check the group available to your account. \n", 23 | "\n", 24 | "If the group was not available for your account, it would show warning logs after the group was set. \n", 25 | "However, the group is invalid, yor might get exceptions when calling a group resources.\n", 26 | "\n", 27 | "### Note: PrimeHubConfig\n", 28 | "\n", 29 | "Please dont update the PrimeHubConfig directly, use the `config` command to that, because only `config` validates for you." 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "id": "167524ce", 35 | "metadata": {}, 36 | "source": [ 37 | "## Setup PrimeHub Python SDK\n" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "id": "2374e1ba", 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "from primehub import PrimeHub, PrimeHubConfig\n", 48 | "ph = PrimeHub(PrimeHubConfig())\n", 49 | "\n", 50 | "if ph.is_ready():\n", 51 | " print(\"PrimeHub Python SDK setup successfully\")\n", 52 | "else:\n", 53 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "e9ad2a22", 59 | "metadata": {}, 60 | "source": [ 61 | "## Help documentation" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "id": "dafdff57", 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "help(ph.config)" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "id": "08dad975", 77 | "metadata": {}, 78 | "source": [ 79 | "## Examples" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "id": "26bc550d", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "# The endpoint was read from config.json, if you want to pick a new endpoint, call the set_endpoint\n", 90 | "ph.config.set_endpoint('https://example.primehub.io/api/graphql')" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "id": "b123a21b", 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "# The API Token was read from config.json, if you want to change a different user, call set_token\n", 101 | "ph.config.set_token('')" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "id": "67b77f0e", 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "# The group was read from config.json, if you want to use resources from a different group, call set_group\n", 112 | "# You might get warning or exceptions during validation.\n", 113 | "ph.config.set_group('another-group')" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "id": "da92b72a", 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [] 123 | } 124 | ], 125 | "metadata": { 126 | "kernelspec": { 127 | "display_name": "Python 3", 128 | "language": "python", 129 | "name": "python3" 130 | }, 131 | "language_info": { 132 | "codemirror_mode": { 133 | "name": "ipython", 134 | "version": 3 135 | }, 136 | "file_extension": ".py", 137 | "mimetype": "text/x-python", 138 | "name": "python", 139 | "nbconvert_exporter": "python", 140 | "pygments_lexer": "ipython3", 141 | "version": "3.7.10" 142 | } 143 | }, 144 | "nbformat": 4, 145 | "nbformat_minor": 5 146 | } 147 | -------------------------------------------------------------------------------- /docs/notebook/groups.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4e2a2222", 6 | "metadata": {}, 7 | "source": [ 8 | "# Groups command\n", 9 | "\n", 10 | "### Introduction\n", 11 | "\n", 12 | "Groups command can show the available groups for your account.\n" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "b41b337d", 18 | "metadata": {}, 19 | "source": [ 20 | "## Setup PrimeHub Python SDK\n" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "2d22406f", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from primehub import PrimeHub, PrimeHubConfig\n", 31 | "ph = PrimeHub(PrimeHubConfig())\n", 32 | "\n", 33 | "if ph.is_ready():\n", 34 | " print(\"PrimeHub Python SDK setup successfully\")\n", 35 | "else:\n", 36 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "15baf8f0", 42 | "metadata": {}, 43 | "source": [ 44 | "## Help documentation" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "3e0ba57d", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "help(ph.groups)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "id": "b0a314a8", 60 | "metadata": {}, 61 | "source": [ 62 | "## Examples" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "id": "26c345f4", 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# Get a group\n", 73 | "ph.groups.get('devteam')" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "id": "e983f14d", 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "# List effective group\n", 84 | "ph.groups.list()" 85 | ] 86 | } 87 | ], 88 | "metadata": { 89 | "kernelspec": { 90 | "display_name": "Python 3 (ipykernel)", 91 | "language": "python", 92 | "name": "python3" 93 | }, 94 | "language_info": { 95 | "codemirror_mode": { 96 | "name": "ipython", 97 | "version": 3 98 | }, 99 | "file_extension": ".py", 100 | "mimetype": "text/x-python", 101 | "name": "python", 102 | "nbconvert_exporter": "python", 103 | "pygments_lexer": "ipython3", 104 | "version": "3.7.6" 105 | } 106 | }, 107 | "nbformat": 4, 108 | "nbformat_minor": 5 109 | } 110 | -------------------------------------------------------------------------------- /docs/notebook/images.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "71088394", 6 | "metadata": {}, 7 | "source": [ 8 | "# Images command\n", 9 | "\n", 10 | "\n", 11 | "The `images` command is one of the group resources command, so it needs to use with a current group.\n" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "40b1d75f", 17 | "metadata": {}, 18 | "source": [ 19 | "## Setup PrimeHub Python SDK\n" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "id": "d9e72f69", 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "from primehub import PrimeHub, PrimeHubConfig\n", 30 | "ph = PrimeHub(PrimeHubConfig())\n", 31 | "\n", 32 | "if ph.is_ready():\n", 33 | " print(\"PrimeHub Python SDK setup successfully\")\n", 34 | "else:\n", 35 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "6b484dbc", 41 | "metadata": {}, 42 | "source": [ 43 | "## Help documentation" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "id": "c05ba53f", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "help(ph.images)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "fb755a49", 59 | "metadata": {}, 60 | "source": [ 61 | "## Examples" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "id": "a963cad4", 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# List images\n", 72 | "ph.images.list()" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "id": "ba7d2f80", 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "# Get an image\n", 83 | "ph.images.get('tf-2')" 84 | ] 85 | } 86 | ], 87 | "metadata": { 88 | "kernelspec": { 89 | "display_name": "Python 3", 90 | "language": "python", 91 | "name": "python3" 92 | }, 93 | "language_info": { 94 | "codemirror_mode": { 95 | "name": "ipython", 96 | "version": 3 97 | }, 98 | "file_extension": ".py", 99 | "mimetype": "text/x-python", 100 | "name": "python", 101 | "nbconvert_exporter": "python", 102 | "pygments_lexer": "ipython3", 103 | "version": "3.7.10" 104 | } 105 | }, 106 | "nbformat": 4, 107 | "nbformat_minor": 5 108 | } 109 | -------------------------------------------------------------------------------- /docs/notebook/instancetypes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b8f26dad", 6 | "metadata": {}, 7 | "source": [ 8 | "# Instancetypes command\n", 9 | "\n", 10 | "\n", 11 | "The `instancetypes` command is one of the group resources command, so it needs to use with a current group.\n" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "d24e7630", 17 | "metadata": {}, 18 | "source": [ 19 | "## Setup PrimeHub Python SDK\n" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "id": "3cdb5c40", 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "from primehub import PrimeHub, PrimeHubConfig\n", 30 | "ph = PrimeHub(PrimeHubConfig())\n", 31 | "\n", 32 | "if ph.is_ready():\n", 33 | " print(\"PrimeHub Python SDK setup successfully\")\n", 34 | "else:\n", 35 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "6b2300e0", 41 | "metadata": {}, 42 | "source": [ 43 | "## Help documentation" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "id": "d055cccd", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "help(ph.instancetypes)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "2d052f37", 59 | "metadata": {}, 60 | "source": [ 61 | "## Examples" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "id": "c3ccfad7", 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# List instanceTypes\n", 72 | "ph.instancetypes.list()" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "id": "477469f7", 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "# Get an instanceType\n", 83 | "ph.instancetypes.get('cpu-1')" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "id": "5976f3ba", 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [] 93 | } 94 | ], 95 | "metadata": { 96 | "kernelspec": { 97 | "display_name": "Python 3", 98 | "language": "python", 99 | "name": "python3" 100 | }, 101 | "language_info": { 102 | "codemirror_mode": { 103 | "name": "ipython", 104 | "version": 3 105 | }, 106 | "file_extension": ".py", 107 | "mimetype": "text/x-python", 108 | "name": "python", 109 | "nbconvert_exporter": "python", 110 | "pygments_lexer": "ipython3", 111 | "version": "3.7.10" 112 | } 113 | }, 114 | "nbformat": 4, 115 | "nbformat_minor": 5 116 | } 117 | -------------------------------------------------------------------------------- /docs/notebook/me.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b7f33587", 6 | "metadata": {}, 7 | "source": [ 8 | "# Me command\n", 9 | "\n", 10 | "### Introduction\n", 11 | "\n", 12 | "Using `me` command to show the account information.\n" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "e0e0168b", 18 | "metadata": {}, 19 | "source": [ 20 | "## Setup PrimeHub Python SDK\n" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "bb88d3e0", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from primehub import PrimeHub, PrimeHubConfig\n", 31 | "ph = PrimeHub(PrimeHubConfig())\n", 32 | "\n", 33 | "if ph.is_ready():\n", 34 | " print(\"PrimeHub Python SDK setup successfully\")\n", 35 | "else:\n", 36 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "f5b3f866", 42 | "metadata": {}, 43 | "source": [ 44 | "## Help documentation" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "e62a8003", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "help(ph.me)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "id": "7c478689", 60 | "metadata": {}, 61 | "source": [ 62 | "## Examples" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "id": "3af5ca83", 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# Show account information\n", 73 | "ph.me.me()" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "id": "13ab410f", 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [] 83 | } 84 | ], 85 | "metadata": { 86 | "kernelspec": { 87 | "display_name": "Python 3", 88 | "language": "python", 89 | "name": "python3" 90 | }, 91 | "language_info": { 92 | "codemirror_mode": { 93 | "name": "ipython", 94 | "version": 3 95 | }, 96 | "file_extension": ".py", 97 | "mimetype": "text/x-python", 98 | "name": "python", 99 | "nbconvert_exporter": "python", 100 | "pygments_lexer": "ipython3", 101 | "version": "3.7.10" 102 | } 103 | }, 104 | "nbformat": 4, 105 | "nbformat_minor": 5 106 | } 107 | -------------------------------------------------------------------------------- /docs/notebook/models.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "697a9c7f", 6 | "metadata": {}, 7 | "source": [ 8 | "# Models command\n", 9 | "\n", 10 | "### Introduction\n", 11 | "\n", 12 | "The `models` command can manage models in the current group.\n", 13 | "\n", 14 | "\n", 15 | "* deploy: Deploy the model version to the speific deployment\n", 16 | "* get: Get the model\n", 17 | "* get-version: Get a version of the model\n", 18 | "* list: List models\n", 19 | "* list-versions: List versions of the model" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "9d26093c", 25 | "metadata": {}, 26 | "source": [ 27 | "## Setup PrimeHub Python SDK\n" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "id": "43fadc2b", 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "from primehub import PrimeHub, PrimeHubConfig\n", 38 | "ph = PrimeHub(PrimeHubConfig())\n", 39 | "\n", 40 | "if ph.is_ready():\n", 41 | " print(\"PrimeHub Python SDK setup successfully\")\n", 42 | "else:\n", 43 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "id": "55a3532f", 49 | "metadata": {}, 50 | "source": [ 51 | "## Help documentation" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "id": "7b042631", 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "help(ph.models)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "id": "d3f7f15b", 67 | "metadata": {}, 68 | "source": [ 69 | "## Examples" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "id": "2f18af7d", 75 | "metadata": {}, 76 | "source": [ 77 | "### Get detail information of models" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "id": "469a3095", 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "# List all models\n", 88 | "models = ph.models.list()\n", 89 | "for model in models:\n", 90 | " print(model)" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "id": "decc1365", 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "# Get detail information by model name\n", 101 | "ph.models.get('')" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "id": "903009c2", 107 | "metadata": {}, 108 | "source": [ 109 | "### Get detail information of model versions" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "id": "1ce5ad39", 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "# List all versions of a model\n", 120 | "model_versions = ph.models.list_versions('')\n", 121 | "for version in model_versions:\n", 122 | " print(version)" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "id": "e0bfb3ad", 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "# Get detail information by model name and version number\n", 133 | "ph.models.get_version('', '')" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "id": "a1970e9c", 139 | "metadata": {}, 140 | "source": [ 141 | "### Deploy a model to existing deployments" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "id": "9464702e", 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "# Deploy the model with specific version \n", 152 | "ph.models.deploy('', '', '')" 153 | ] 154 | } 155 | ], 156 | "metadata": { 157 | "kernelspec": { 158 | "display_name": "Python 3", 159 | "language": "python", 160 | "name": "python3" 161 | }, 162 | "language_info": { 163 | "codemirror_mode": { 164 | "name": "ipython", 165 | "version": 3 166 | }, 167 | "file_extension": ".py", 168 | "mimetype": "text/x-python", 169 | "name": "python", 170 | "nbconvert_exporter": "python", 171 | "pygments_lexer": "ipython3", 172 | "version": "3.7.10" 173 | } 174 | }, 175 | "nbformat": 4, 176 | "nbformat_minor": 5 177 | } 178 | -------------------------------------------------------------------------------- /docs/notebook/notebooks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "87382334", 6 | "metadata": {}, 7 | "source": [ 8 | "# Notebooks command\n", 9 | "\n", 10 | "### Introduction\n", 11 | "\n", 12 | "The `notebooks` command can manage your notebook." 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "d7e67662", 18 | "metadata": {}, 19 | "source": [ 20 | "## Setup PrimeHub Python SDK\n" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "fef8aff5", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from primehub import PrimeHub, PrimeHubConfig\n", 31 | "ph = PrimeHub(PrimeHubConfig())\n", 32 | "\n", 33 | "if ph.is_ready():\n", 34 | " print(\"PrimeHub Python SDK setup successfully\")\n", 35 | "else:\n", 36 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "ae63b2af", 42 | "metadata": {}, 43 | "source": [ 44 | "## Help documentation" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "7c8e9e31", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "help(ph.notebooks)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "id": "45f1a0e5", 60 | "metadata": {}, 61 | "source": [ 62 | "## Examples" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "id": "96a0c2fb", 68 | "metadata": {}, 69 | "source": [ 70 | "### Get detail information of notebook" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "id": "fd9f0a64", 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "# Get logs\n", 81 | "logs = ph.notebooks.logs()\n", 82 | "for l in logs:\n", 83 | " print(l)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "id": "8fef781f", 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "# Get logs in stream mode\n", 94 | "logs = ph.notebooks.logs(follow=True)\n", 95 | "for l in logs:\n", 96 | " print(l)" 97 | ] 98 | } 99 | ], 100 | "metadata": { 101 | "kernelspec": { 102 | "display_name": "Python 3", 103 | "language": "python", 104 | "name": "python3" 105 | }, 106 | "language_info": { 107 | "codemirror_mode": { 108 | "name": "ipython", 109 | "version": 3 110 | }, 111 | "file_extension": ".py", 112 | "mimetype": "text/x-python", 113 | "name": "python", 114 | "nbconvert_exporter": "python", 115 | "pygments_lexer": "ipython3", 116 | "version": "3.7.10" 117 | } 118 | }, 119 | "nbformat": 4, 120 | "nbformat_minor": 5 121 | } 122 | -------------------------------------------------------------------------------- /docs/notebook/secrets.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "6f94585c", 6 | "metadata": {}, 7 | "source": [ 8 | "# Secrets command\n", 9 | "\n", 10 | "### Introduction\n", 11 | "\n", 12 | "Secrets command can show secret information. You will use the `id` of a `secret` with features that support **Image Pull Secret**.\n" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "e035ff8a", 18 | "metadata": {}, 19 | "source": [ 20 | "## Setup PrimeHub Python SDK\n" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "14d61c20", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from primehub import PrimeHub, PrimeHubConfig\n", 31 | "ph = PrimeHub(PrimeHubConfig())\n", 32 | "\n", 33 | "if ph.is_ready():\n", 34 | " print(\"PrimeHub Python SDK setup successfully\")\n", 35 | "else:\n", 36 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "99b72e74", 42 | "metadata": {}, 43 | "source": [ 44 | "## Help documentation" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "5c19aff9", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "help(ph.secrets)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "id": "fa526193", 60 | "metadata": {}, 61 | "source": [ 62 | "## Examples" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "id": "99019b63", 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# List all secrets\n", 73 | "secrets = list(ph.secrets.list())\n", 74 | "secrets" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "id": "40162ded", 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "# Get a secret\n", 85 | "id = secrets[0]['id'] if secrets else None\n", 86 | "ph.secrets.get(id)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "id": "e97cc2f6", 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [] 96 | } 97 | ], 98 | "metadata": { 99 | "kernelspec": { 100 | "display_name": "Python 3 (ipykernel)", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.7.6" 115 | } 116 | }, 117 | "nbformat": 4, 118 | "nbformat_minor": 5 119 | } 120 | -------------------------------------------------------------------------------- /docs/notebook/volumes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "3365d96c", 6 | "metadata": {}, 7 | "source": [ 8 | "# Volumes command\n", 9 | "\n", 10 | "\n", 11 | "The `volumes` command is one of the group resources command, so it needs to use with a current group.\n" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "0e684e90", 17 | "metadata": {}, 18 | "source": [ 19 | "## Setup PrimeHub Python SDK\n" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "id": "2ed0bf22", 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "from primehub import PrimeHub, PrimeHubConfig\n", 30 | "ph = PrimeHub(PrimeHubConfig())\n", 31 | "\n", 32 | "if ph.is_ready():\n", 33 | " print(\"PrimeHub Python SDK setup successfully\")\n", 34 | "else:\n", 35 | " print(\"PrimeHub Python SDK couldn't get the group information, follow the 00-getting-started.ipynb to complete it\")" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "3720ad82", 41 | "metadata": {}, 42 | "source": [ 43 | "## Help documentation" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "id": "2055801e", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "help(ph.volumes)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "dcb77e3d", 59 | "metadata": {}, 60 | "source": [ 61 | "## Examples" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "id": "639e6281", 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# List volumes\n", 72 | "ph.volumes.list()" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "id": "908759a6", 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "# Get a volume\n", 83 | "ph.volumes.get('primehub')" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "id": "4b9f9704", 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [] 93 | } 94 | ], 95 | "metadata": { 96 | "kernelspec": { 97 | "display_name": "Python 3", 98 | "language": "python", 99 | "name": "python3" 100 | }, 101 | "language_info": { 102 | "codemirror_mode": { 103 | "name": "ipython", 104 | "version": 3 105 | }, 106 | "file_extension": ".py", 107 | "mimetype": "text/x-python", 108 | "name": "python", 109 | "nbconvert_exporter": "python", 110 | "pygments_lexer": "ipython3", 111 | "version": "3.7.10" 112 | } 113 | }, 114 | "nbformat": 4, 115 | "nbformat_minor": 5 116 | } 117 | -------------------------------------------------------------------------------- /docs/screenplay/README.md: -------------------------------------------------------------------------------- 1 | # PrimeHub Python SDK screenplay 2 | 3 | PrimeHub Python SDK screenplay is a set of Juypter notebooks for user to test PrimeHub SDK function. 4 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | 3 | [mypy-primehub.utils.argparser] 4 | ignore_errors = True 5 | 6 | [mypy-tests.*] 7 | ignore_errors = True 8 | -------------------------------------------------------------------------------- /primehub-sdk-autocomplete.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | autoload -U bashcompinit 3 | 4 | _primehub() { 5 | COMPREPLY=($(cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" auto-primehub)) 6 | } 7 | 8 | complete -F _primehub primehub 9 | -------------------------------------------------------------------------------- /primehub/VERSION: -------------------------------------------------------------------------------- 1 | 0.3.0-dev 2 | -------------------------------------------------------------------------------- /primehub/admin_reports.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | from typing import Iterator 3 | 4 | from primehub import Helpful, Module, PrimeHubException, cmd 5 | 6 | 7 | class AdminReport(Helpful, Module): 8 | 9 | @cmd(name='download', description='Download a report by url', optionals=[('dest', str)]) 10 | def download(self, url, **kwargs): 11 | """ 12 | Download a report csv file from the given url. 13 | 14 | It will convert the URI to filename by default. For example, there are summary url and details url 15 | * https://primehub-python-sdk.primehub.io/api/report/monthly/2022/12 16 | * https://primehub-python-sdk.primehub.io/api/report/monthly/details/2022/12 17 | 18 | Will save to 19 | * 202212.csv 20 | * 202212_details.csv 21 | 22 | If you give a dest, it will use the given dest as the filename. 23 | 24 | :type url: str 25 | :param url: The report url. 26 | 27 | :type dest: str 28 | :param dest: The local path to save the report csv file 29 | 30 | :type recusive: bool 31 | :param recusive: Copy recursively, it works when a path is a directory. 32 | """ 33 | 34 | def convert_to_filename(url, segment_str, dest): 35 | if dest: 36 | return os.path.abspath(dest) 37 | filename = url.split(segment_str)[1].replace('/', '') 38 | if 'details' in segment_str: 39 | return f'{filename}_details.csv' 40 | return os.path.join(os.getcwd(), f'{filename}.csv') 41 | 42 | dest = kwargs.get('dest') 43 | filename = None 44 | if '/monthly/details/' in url: 45 | filename = convert_to_filename(url, '/monthly/details/', dest) 46 | elif '/monthly/' in url: 47 | filename = convert_to_filename(url, '/monthly/', dest) 48 | else: 49 | raise PrimeHubException(f'invalid url: {url}') 50 | 51 | def prepare_directory(dest: str): 52 | try: 53 | os.truncate(dest, 0) 54 | return 55 | except BaseException: 56 | pass 57 | 58 | try: 59 | os.makedirs(os.path.dirname(dest), exist_ok=True) 60 | except BaseException: 61 | pass 62 | 63 | prepare_directory(filename) 64 | self.request_file(url, filename) 65 | return dict(filename=filename) 66 | 67 | @cmd(name='list', description='List reports', optionals=[('page', int)]) 68 | def list(self, **kwargs) -> Iterator: 69 | """ 70 | List reports 71 | 72 | :type page: int 73 | :param page: the page of all data 74 | 75 | :rtype Iterator 76 | :return user iterator 77 | """ 78 | 79 | query = """ 80 | query UsageReportQuery($usageReportPage: Int) { 81 | usageReport: usageReportsConnection(page: $usageReportPage) { 82 | edges { 83 | cursor 84 | node { 85 | id 86 | summaryUrl 87 | detailedUrl 88 | } 89 | } 90 | pageInfo { 91 | currentPage 92 | totalPage 93 | } 94 | } 95 | } 96 | """ 97 | 98 | variables = {'usageReportPage': 1} 99 | 100 | page = kwargs.get('page', 1) 101 | if page >= 1: 102 | variables['usageReportPage'] = int(page) 103 | results = self.request(variables, query) 104 | if results['data']['usageReport']['edges']: 105 | for e in results['data']['usageReport']['edges']: 106 | yield e['node'] 107 | return 108 | 109 | page = 1 110 | while True: 111 | variables['usageReportPage'] = int(page) 112 | results = self.request(variables, query) 113 | if results['data']['users']['edges']: 114 | for e in results['data']['users']['edges']: 115 | yield e['node'] 116 | page = page + 1 117 | else: 118 | break 119 | 120 | def help_description(self): 121 | return "Get reports" 122 | -------------------------------------------------------------------------------- /primehub/apptemplates.py: -------------------------------------------------------------------------------- 1 | from typing import Iterator, Any 2 | 3 | from primehub import Helpful, cmd, Module 4 | from primehub.utils.display import display_tree_like_format 5 | 6 | 7 | class AppTemplate(Helpful, Module): 8 | 9 | @cmd(name='list', description='List PhApp templates', return_required=True) 10 | def list(self) -> Iterator: 11 | """ 12 | List PhApp templates 13 | 14 | :rtype: Iterator 15 | :returns: PhApp templates 16 | """ 17 | query = """ 18 | query GetPhAppTemplates { 19 | phAppTemplates { 20 | id 21 | name 22 | icon 23 | version 24 | description 25 | docLink 26 | template 27 | } 28 | } 29 | """ 30 | result = self.request({}, query) 31 | if 'data' in result and 'phAppTemplates' in result['data']: 32 | result = result['data']['phAppTemplates'] 33 | for template in result: 34 | yield template 35 | return result 36 | 37 | @cmd(name='get', description='Get a PhApp template', return_required=True) 38 | def get(self, id: str) -> dict: 39 | """ 40 | Get a PhApp template 41 | 42 | :rtype: dict 43 | :returns: a PhApp template 44 | """ 45 | query = """ 46 | query GetPhAppTemplates($where: PhAppTemplateWhereInput) { 47 | phAppTemplates(where: $where) { 48 | id 49 | name 50 | icon 51 | version 52 | description 53 | docLink 54 | template 55 | defaultEnvs { 56 | name 57 | defaultValue 58 | optional 59 | description 60 | } 61 | } 62 | } 63 | """ 64 | result = self.request({'where': {'id': id}}, query) 65 | if 'data' in result and 'phAppTemplates' in result['data']: 66 | result = result['data']['phAppTemplates'] 67 | return result[0] 68 | return result 69 | 70 | def help_description(self): 71 | return "Get PhAppTemplates" 72 | 73 | def display(self, action: dict, value: Any): 74 | # customize the list view from columns to tree-like 75 | if action['func'] == AppTemplate.list.__name__ and self.get_display().name != 'json': 76 | for template in value: 77 | template = self.convert_for_human_friendly_data(template) 78 | display_tree_like_format(template, file=self.primehub.stdout) 79 | print("", file=self.primehub.stdout) 80 | elif action['func'] == AppTemplate.get.__name__ and self.get_display().name != 'json': 81 | super(AppTemplate, self).display(action, self.convert_for_human_friendly_data(value)) 82 | else: 83 | super(AppTemplate, self).display(action, value) 84 | 85 | def convert_for_human_friendly_data(self, template): 86 | # drop template and icon for human-friendly view 87 | t = template.pop('template') 88 | template.pop('icon') 89 | if 'defaultEnvs' in template: 90 | template.pop('defaultEnvs') 91 | 92 | image = t.get('spec', {}).get('podTemplate', {}).get('spec', {}).get('containers', [{}])[0].get('image') 93 | template['image'] = image 94 | return template 95 | -------------------------------------------------------------------------------- /primehub/extras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InfuseAI/primehub-python-sdk/5cde4c72a42750d76f93d03c8cce818ba855ea30/primehub/extras/__init__.py -------------------------------------------------------------------------------- /primehub/extras/devlab.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | import time 4 | from tempfile import mkstemp 5 | 6 | from primehub import Helpful, cmd, has_data_from_stdin, Module 7 | from primehub.utils.permission import ask_for_permission 8 | 9 | 10 | class DevLab(Helpful, Module): 11 | 12 | @cmd(name='submit-case', description='submit use case') 13 | def read_from_stdin_or_file(self, *args, **kwargs): 14 | """ 15 | primehub devlab submit-case abc -f -xd < {}".format(last_state, p)) 85 | last_state = p 86 | 87 | if last_state == 'Running': 88 | break 89 | 90 | if last_state == 'Succeeded': 91 | break 92 | time.sleep(1) 93 | 94 | print("Logs:") 95 | for g in self.primehub.jobs.logs(my_id, follow=True): 96 | print(g) 97 | 98 | def help_description(self): 99 | return "dev-lab is used to the primehub-python-sdk development and testing" 100 | -------------------------------------------------------------------------------- /primehub/extras/doc_generator.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import re 4 | import sys 5 | 6 | from jinja2 import Environment, PackageLoader, select_autoescape 7 | 8 | from primehub import PrimeHub 9 | from primehub.cli import create_sdk, main as cli_main 10 | from primehub.utils.decorators import find_actions, find_action_method 11 | 12 | env = Environment( 13 | loader=PackageLoader("primehub.extras"), 14 | autoescape=select_autoescape() 15 | ) 16 | 17 | 18 | def get_example(command, role=''): 19 | try: 20 | if role: 21 | return env.get_template('examples/{}/{}.md'.format(role, command)).render() 22 | else: 23 | return env.get_template('examples/{}.md'.format(command)).render() 24 | except BaseException: 25 | pass 26 | return "TBD: please write example for [{}]".format(command) 27 | 28 | 29 | def get_doc_path(): 30 | import primehub 31 | 32 | p = os.path.abspath(os.path.dirname(primehub.__file__) + "/../docs") 33 | return p 34 | 35 | 36 | def create_cli_doc_path(name, role=''): 37 | if role: 38 | doc_path = os.path.join(get_doc_path(), 'CLI', role, name + ".md") 39 | else: 40 | doc_path = os.path.join(get_doc_path(), 'CLI', name + ".md") 41 | os.makedirs(os.path.dirname(doc_path), exist_ok=True) 42 | return doc_path 43 | 44 | 45 | def generate_command_document(*args, **kwargs): 46 | if kwargs['role']: 47 | kwargs['role_title'] = f'<{kwargs["role"].upper()}> ' 48 | kwargs['role'] = f'{kwargs["role"]} ' 49 | 50 | return env.get_template('cli.tpl.md').render(*args, **kwargs) 51 | 52 | 53 | def generate_help_for_command(sdk: PrimeHub, name, role=''): 54 | sdk.stderr = io.StringIO() 55 | sdk.stdout = io.StringIO() 56 | 57 | if role: 58 | sys.argv = ['primehub', role, name, '-h'] 59 | else: 60 | sys.argv = ['primehub', name, '-h'] 61 | try: 62 | cli_main(sdk=sdk) 63 | except SystemExit: 64 | pass 65 | command_help = sdk.stderr.getvalue() 66 | actions = find_actions(sdk.commands[name]) 67 | attach_template_information_to_action(actions, name, sdk) 68 | document = generate_command_document(command=name, command_help=command_help, role=role, 69 | actions=actions, examples=get_example(name, role)) 70 | 71 | print("Generate doc", name) 72 | p = create_cli_doc_path(name, role) 73 | with open(p, "w") as fh: 74 | fh.write(document) 75 | 76 | 77 | def attach_template_information_to_action(actions, name, sdk): 78 | for action in actions: 79 | explain = extract_description_from_docstring(action, name, sdk) 80 | 81 | def explained(x): 82 | if x in explain: 83 | return "{}: {}".format(x, explain[x]) 84 | return x 85 | 86 | # arguments 87 | arg_list = [] 88 | for x in action['arguments']: 89 | if x[2] is True: 90 | # skip **kwargs 91 | continue 92 | arg_list.append(x[0]) 93 | action['required_arguments_string'] = " ".join(["<%s>" % x for x in arg_list]) 94 | action['required_arguments'] = [explained(x) for x in arg_list] 95 | 96 | # optionals 97 | opt_list = [] 98 | for x in action['optionals']: 99 | opt_list.append(x[0]) 100 | action['optional_arguments'] = [explained(x) for x in opt_list] 101 | 102 | 103 | def extract_description_from_docstring(action, name, sdk): 104 | output = dict() 105 | method_name = find_action_method(sdk.commands[name], action['name']) 106 | doc_string = getattr(sdk.commands[name], method_name).__doc__ 107 | param_description = re.findall(r':param ([^:]+):(.+)', str(doc_string)) 108 | for k, v in param_description: 109 | output[k.strip()] = v.strip() 110 | return output 111 | 112 | 113 | def main(): 114 | sdk = create_sdk() 115 | for k, v in sdk.commands.items(): 116 | if k == 'devlab': 117 | continue 118 | if k == 'version': 119 | continue 120 | if k == 'admin': 121 | continue 122 | generate_help_for_command(sdk, k) 123 | 124 | for k, v in sdk.admin_commands.items(): 125 | generate_help_for_command(sdk, k, 'admin') 126 | 127 | 128 | if __name__ == '__main__': 129 | main() 130 | -------------------------------------------------------------------------------- /primehub/extras/e2e.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | from tempfile import mkstemp 4 | 5 | from primehub import Helpful, cmd, Module 6 | from primehub.config import Config 7 | from primehub.groups import Groups 8 | from primehub.utils import create_logger 9 | 10 | logger = create_logger('e2e') 11 | 12 | 13 | class E2EForBasicFunction(Helpful, Module): 14 | 15 | def title(self, name): 16 | print("{} {:15s} {}".format('=' * 30, name, '=' * 30)) 17 | 18 | def endline(self): 19 | print() 20 | print() 21 | 22 | @cmd(name='basic-functions', description='run basic functions') 23 | def basic_functions(self): 24 | self.title('Config') 25 | c: Config = self.primehub.config 26 | c.primehub.primehub_config.save(path='./e2e-config.json') 27 | print('save config to ', './e2e-config.json') 28 | self.endline() 29 | 30 | self.title('Info') 31 | print(self.primehub.info.info()) 32 | self.endline() 33 | 34 | self.title('Groups') 35 | group: Groups = self.primehub.groups 36 | for g in group.list(): 37 | print(g['id'], g['name']) 38 | self.endline() 39 | 40 | self.title('Jobs') 41 | self.job_e2e() 42 | 43 | def job_e2e(self): 44 | instance_types = [x for x in self.primehub.instancetypes.list() if "gpu" not in x] 45 | instance_type = None 46 | for x in instance_types: 47 | if 'cpu-half' in x['name']: 48 | instance_type = x['id'] 49 | break 50 | if 'cpu-1' in x['name']: 51 | instance_type = x['id'] 52 | break 53 | print('instance-type:', instance_type) 54 | scripts = r""" 55 | sudo apt-get update -y 56 | sudo apt-get install -y git 57 | pip install git+https://github.com/InfuseAI/primehub-python-sdk.git@main 58 | primehub -h 59 | echo 60 | exit 0 61 | """ 62 | job_spec = dict(instanceType=instance_type, image="base-notebook", displayName="job-e2e", command=scripts) 63 | fd, path = mkstemp(".json") 64 | with open(path, "w") as fh: 65 | fh.write(json.dumps(job_spec)) 66 | 67 | my_job = self.primehub.jobs._submit_cmd(file=path) 68 | my_id = my_job['id'] 69 | print("Job ID:", my_job['id']) 70 | 71 | last_state = None 72 | while True: 73 | p = self.primehub.jobs.get(my_id)['phase'] 74 | if last_state is None: 75 | last_state = p 76 | 77 | if p != last_state: 78 | print("Job Phase: {} -> {}".format(last_state, p)) 79 | last_state = p 80 | 81 | if last_state == 'Running': 82 | break 83 | 84 | if last_state == 'Succeeded': 85 | break 86 | 87 | if last_state == 'Cancelled': 88 | break 89 | time.sleep(1) 90 | 91 | print("Logs:") 92 | for g in self.primehub.jobs.logs(my_id, follow=True): 93 | print(g) 94 | 95 | def help_description(self): 96 | return "dev-lab is used to the primehub-python-sdk development and testing" 97 | -------------------------------------------------------------------------------- /primehub/extras/templates/cli.tpl.md: -------------------------------------------------------------------------------- 1 | 2 | # {{role_title}}Primehub {{command.capitalize()}} 3 | 4 | ``` 5 | {{command_help}} 6 | ``` 7 | 8 | {% for item in actions %} 9 | ### {{item['name']}} 10 | 11 | {{item['description']}} 12 | 13 | {% if item['required_arguments'] %} 14 | ``` 15 | primehub {{role}}{{command}} {{item['name']}} {{item['required_arguments_string']}} 16 | ``` 17 | {% else %} 18 | ``` 19 | primehub {{role}}{{command}} {{item['name']}} 20 | ``` 21 | {% endif %} 22 | 23 | {%- if item['required_arguments'] -%} 24 | {%- for argument in item['required_arguments'] %} 25 | * {{argument}} 26 | {%- endfor %} 27 | {% endif %} {# end of :: if item['required_arguments'] #} 28 | {% for argument in item['optional_arguments'] %} 29 | * *(optional)* {{argument}} 30 | {% endfor %} 31 | 32 | 33 | {% endfor %} {# end of :: for item in actions #} 34 | 35 | ## Examples 36 | 37 | {{examples}} 38 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/admin/reports.md: -------------------------------------------------------------------------------- 1 | 2 | ### Reports list and download 3 | 4 | List reports 5 | 6 | ``` 7 | $ primehub admin reports list 8 | primehub admin reports list --json | jq . 9 | [ 10 | { 11 | "id": "2022/12", 12 | "summaryUrl": "https://primehub-python-sdk.primehub.io/api/report/monthly/2022/12", 13 | "detailedUrl": "https://primehub-python-sdk.primehub.io/api/report/monthly/details/2022/12" 14 | } 15 | ] 16 | ``` 17 | 18 | Download a report by url: 19 | 20 | ``` 21 | $ primehub admin reports download https://primehub-python-sdk.primehub.io/api/report/monthly/details/2022/12 22 | filename: 202212_details.csv 23 | ``` 24 | 25 | Download a report with `--dest` to change the download path: 26 | 27 | ``` 28 | $ primehub admin reports download https://primehub-python-sdk.primehub.io/api/report/monthly/2022/12 --dest foo/bar/202212.csv 29 | filename: /home/primehub/foo/bar/202212.csv 30 | ``` 31 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/admin/secrets.md: -------------------------------------------------------------------------------- 1 | ### Fields for creating or updating 2 | 3 | | field | required | type | description | 4 | | --- | --- | --- | --- | 5 | | name | required | string | The name of secret. It is only used when creating. | 6 | | type | required | string | one of ['opaque', 'kubernetes']. `opaque` is used for Git Sync Volume (SSH Public Key). `kubernetes` is used for Container Registry. | 7 | | displayName | optional | string | | 8 | 9 | * `type` can not be changed after created. 10 | 11 | Fields for `opaque` 12 | 13 | | field | required | type | description | 14 | | --- | --- | --- | --- | 15 | | secret | conditional | string | when type is opaque, secret field become required for the SSH Public Key. | 16 | 17 | Fields for `kubernetes` 18 | 19 | You should put container registry credentials to these fields: 20 | 21 | | field | required | type | description | 22 | | --- | --- | --- | --- | 23 | | registryHost | conditional | string | | 24 | | username | conditional | string | | 25 | | password | conditional | string | | 26 | 27 | ### Create secrets 28 | 29 | Create a secret for a Git Sync volume 30 | 31 | ``` 32 | primehub admin secrets create < Be careful with the config command, it will `MODIFY` the configuration file directly 5 | 6 | ### Example: create a config file 7 | 8 | We could use the `config` command with the `--config` flag to make a new config to the `my.config` path. 9 | 10 | ``` 11 | primehub config set-endpoint http://primehub-python-sdk.primehub.io --config my-config.json 12 | primehub config set-token my-token --config my-config.json 13 | primehub config set-group no-such-group --config my-config.json 14 | ``` 15 | 16 | You might see the warning, because of the fake endpoint: 17 | 18 | ``` 19 | 2021-08-03 11:05:17,209 - cmd-config - WARNING - Got request problem with http://primehub-python-sdk.primehub.io: HTTPConnectionPool(host='primehub-python-sdk.primehub.io', port=80): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')) 20 | No group information for [no-such-group], please set the right group 21 | ``` 22 | 23 | However, `config` still create the file for you: 24 | 25 | ```json 26 | { 27 | "endpoint": "http://primehub-python-sdk.primehub.io", 28 | "api-token": "my-token", 29 | "group": { 30 | "name": null 31 | } 32 | } 33 | ``` 34 | 35 | The SDK can not fetch the group information from the server, finally you got a group with null name. If you did 36 | configure with a real api server, the group information would be available: 37 | 38 | ```json 39 | { 40 | "endpoint": "http://primehub-python-sdk.primehub.io", 41 | "api-token": "my-token", 42 | "group": { 43 | "id": "927406f6-88b0-490e-8e36-7835224fdf13", 44 | "name": "sdk", 45 | "displayName": "PrimeHub Python SDK Team" 46 | } 47 | } 48 | ``` 49 | 50 | ### Example: generate API Token with OAuth login 51 | 52 | ``` 53 | primehub config generate-token http://primehub-python-sdk.primehub.io 54 | ``` 55 | 56 | The commands will guide you how to get an API token, it will show 2 required actions: 57 | * visit the URL and login with your account and copy the authorization code from the web page 58 | * paste the authorization code back to the terminal 59 | 60 | 61 | ``` 62 | Go to this URL in the browser http://primehub-python-sdk.primehub.io/console/oidc/auth-flow/request 63 | Enter your authorization code: 64 | ``` 65 | 66 | When all actions have done, the configuration got updated: 67 | 68 | ``` 69 | Found old configuration, backup it to /home/phadmin/.primehub/config-20211001161504.json 70 | PrimeHub SDK Config has been updated: /home/phadmin/.primehub/config.json 71 | ``` 72 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/files.md: -------------------------------------------------------------------------------- 1 | We could use `list` to watch files or directory remotely. 2 | 3 | `list` requires a `path` parameter, it always starts with `/`: 4 | 5 | ``` 6 | $ primehub files list / 7 | name size lastModified phfsUri 8 | ------------- ------ -------------- --------------------- 9 | jobArtifacts/ 0 phfs:///jobArtifacts/ 10 | ``` 11 | 12 | You might go deeply into a sub-directory: 13 | 14 | ``` 15 | $ primehub files list /jobArtifacts/job-202107290838-aoq173/ 16 | name size lastModified phfsUri 17 | ---------- ------ -------------- ------------------ 18 | .metadata/ 0 phfs:///.metadata/ 19 | ``` 20 | 21 | Then `download` a file directly or a directory with `--recursive` options 22 | 23 | ``` 24 | $ primehub files download /jobArtifacts/job-202107290838-aoq173 ./my-download --recursive 25 | ``` 26 | 27 | ``` 28 | ./my-download 29 | └── job-202107290838-aoq173 30 | └── .metadata 31 | 32 | 1 directory, 1 file 33 | ``` 34 | 35 | Uses `upload` to upload a file or a directory with `--recursive` options. 36 | 37 | *Note: it follows shell `cp -R` manner when upload a directory* 38 | > If the source_file ends in a /, the contents of the directory are copied rather than the directory itself. 39 | 40 | ``` 41 | $ primehub files upload ./my-download /jobArtifacts/job-202107290838-aoq173 --recursive 42 | [Uploading] ./my-download/job-202107290838-aoq173/.metadata -> phfs:///jobArtifacts/job-202107290838-aoq173/my-download/job-202107290838-aoq173/.metadata 43 | success phfs file 44 | --------- ----------------------------------------------------------------------------------- ----------------------------------------------- 45 | True /jobArtifacts/job-202107290838-aoq173/my-download/job-202107290838-aoq173/.metadata ./my-download/job-202107290838-aoq173/.metadata 46 | ``` 47 | 48 | ``` 49 | $ primehub files upload ./my-download/ /jobArtifacts/job-202107290838-aoq173 --recursive 50 | [Uploading] ./my-download/job-202107290838-aoq173/.metadata -> phfs:///jobArtifacts/job-202107290838-aoq173/job-202107290838-aoq173/.metadata 51 | success phfs file 52 | --------- ----------------------------------------------------------------------- ----------------------------------------------- 53 | True /jobArtifacts/job-202107290838-aoq173/job-202107290838-aoq173/.metadata ./my-download/job-202107290838-aoq173/.metadata 54 | ``` 55 | 56 | Uses `delete` to delete a file or a directory with `--recursive` options. 57 | 58 | ``` 59 | $ primehub files delete /jobArtifacts/job-202107290838-aoq173 --recursive 60 | deleteFiles: 3 61 | ``` 62 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/groups.md: -------------------------------------------------------------------------------- 1 | `groups` command can show the available groups for your account. 2 | 3 | If you have the name of a group, `get` will be useful: 4 | 5 | ``` 6 | primehub groups get phusers 7 | ``` 8 | 9 | The `list` command will show any groups your account belongs to: 10 | 11 | ``` 12 | primehub groups list 13 | ``` 14 | 15 | ``` 16 | id name displayName quotaCpu quotaGpu quotaMemory projectQuotaCpu projectQuotaGpu projectQuotaMemory 17 | ------------------------------------ ------- -------------- ---------- ---------- ------------- ----------------- ----------------- -------------------- 18 | 2b080113-e2f1-4b1b-a6ef-eb0ca5e2f376 phusers primehub users 0 0 19 | ``` 20 | 21 | ### Group members 22 | 23 | List group members of a group by id 24 | 25 | ``` 26 | $ primehub groups list-users daefae90-0fc7-4a5f-ab2c-9a193c461225 27 | 28 | id username firstName lastName email group_admin 29 | ------------------------------------ -------------------- ----------- ---------- -------------------- ------------- 30 | 9e26cfc4-faba-4aa5-85a8-e8da93eb1a38 foobar Foo Bar hi@infuseai.io False 31 | ``` 32 | 33 | Add a member to a group by id 34 | 35 | ``` 36 | $ primehub groups add-user daefae90-0fc7-4a5f-ab2c-9a193c461225 4ca09a19-ef14-4800-861f-aec74149a6f4 37 | ``` 38 | * Add `--is_admin` flag to grant the member group admin permission 39 | 40 | Remove a member from a group by id 41 | 42 | ``` 43 | $ primehub groups remove-user daefae90-0fc7-4a5f-ab2c-9a193c461225 4ca09a19-ef14-4800-861f-aec74149a6f4 44 | ``` 45 | 46 | ### MLflow configuration 47 | 48 | To get MLflow configuration of a group, use `get-mlflow` command: 49 | 50 | ``` 51 | primehub groups get-mlflow 52 | ``` 53 | 54 | ``` 55 | trackingUri: http://app-mlflow-xyzab:5000 56 | uiUrl: https://primehub-python-sdk.primehub.io/console/apps/mlflow-xyzab 57 | trackingEnvs: [{'name': 'NAME1', 'value': 'value1'}] 58 | artifactEnvs: [] 59 | ``` 60 | 61 | To set MLflow configuration, use `set-mlflow` command: 62 | 63 | ```bash 64 | primehub group set-mlflow < --file /tmp/mlflow.json 93 | ``` 94 | 95 | To clear MLflow for a group, use `unset-mlflow` command: 96 | 97 | ``` 98 | primehub groups unset-mlflow 99 | ``` 100 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/images.md: -------------------------------------------------------------------------------- 1 | The `images` command is a group specific resource. It only works after the `group` assigned. 2 | 3 | Using `list` to find all images in your group: 4 | 5 | ``` 6 | primehub images list 7 | ``` 8 | 9 | ```name displayName description 10 | ------------- ------------------------------ ------------------------------ 11 | pytorch-1 PyTorch 1.8.0 (Python 3.7) PyTorch 1.8.0 (Python 3.7) 12 | tf-2 TensorFlow 2.5.0 (Python 3.7) TensorFlow 2.5.0 (Python 3.7) 13 | base-notebook base-notebook base notebook 14 | tf-1 TensorFlow 1.15.4 (Python 3.7) TensorFlow 1.15.4 (Python 3.7 15 | ``` 16 | 17 | If you already know the name of a images, use the `get` to get a single entry: 18 | 19 | ``` 20 | primehub images get tf-2 21 | ``` 22 | 23 | ``` 24 | id: tf-2 25 | name: tf-2 26 | displayName: TensorFlow 2.5.0 (Python 3.7) 27 | description: TensorFlow 2.5.0 (Python 3.7) 28 | useImagePullSecret: None 29 | spec: 30 | description: TensorFlow 2.5.0 (Python 3.7) 31 | displayName: TensorFlow 2.5.0 (Python 3.7) 32 | type: both 33 | url: infuseai/docker-stacks:tensorflow-notebook-v2-5-0-63fdf50a 34 | urlForGpu: infuseai/docker-stacks:tensorflow-notebook-v2-5-0-63fdf50a-gpu-cuda-11 35 | ``` 36 | 37 | Create a group image: 38 | 39 | ``` 40 | $ primehub images create < 53 | primehub info info 54 | ``` 55 | 56 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/instancetypes.md: -------------------------------------------------------------------------------- 1 | The `instancetypes` command is a group specific resource. It only works after the `group` assigned. 2 | 3 | Using `list` to find all instance types in your group: 4 | 5 | ``` 6 | primehub instancetypes list 7 | ``` 8 | 9 | ``` 10 | name displayName description cpuRequest cpuLimit memoryRequest memoryLimit gpuLimit global 11 | -------------------------------- --------------------------------------------- ------------------------------ ------------ ---------- --------------- ------------- ---------- -------- 12 | cpu-2 CPU 2 2 CPU / 10G 2 10 0 True 13 | gpu-1 GPU 1 2 CPU / 7G / 1 GPU 1.5 2 5 7 1 True 14 | gpu-2 GPU 2 4 CPU / 14G / 2 GPU 4 4 14 14 2 True 15 | cpu-1 CPU 1 1 CPU / 2G 1 1 2 2 0 True 16 | ``` 17 | 18 | If you already know the name of an instance type, use the `get` to get a single entry: 19 | 20 | ``` 21 | primehub instancetypes get cpu-1 22 | ``` 23 | 24 | ``` 25 | name: cpu-1 26 | displayName: CPU 1 27 | description: 1 CPU / 2G 28 | cpuRequest: 1 29 | cpuLimit: 1 30 | memoryRequest: 2 31 | memoryLimit: 2 32 | gpuLimit: 0 33 | global: True 34 | tolerations: [{'operator': 'Equal', 'key': 'hub.jupyter.org/dedicated', 'value': 'user', 'effect': 'NoSchedule'}] 35 | nodeSelector: None 36 | ``` 37 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/jobs.md: -------------------------------------------------------------------------------- 1 | ### Fields for submitting 2 | 3 | | field | required | type | description | 4 | | --- | --- | --- | --- | 5 | | displayName | required | string | display name | 6 | | instanceType | required | string | instance type which allocates resources for the job | 7 | | image | required | string | image which the job run bases on | 8 | | command | required | string | sequential commands of the job context | 9 | | activeDeadlineSeconds | optional | int | a running job will be cancelled after this time period (in seconds) | 10 | 11 | ### Submit a job 12 | 13 | Using `submit` to create a job, it needs the job definition. We could give the job definition either `stdin` or `--file` 14 | flag. 15 | 16 | ``` 17 | $ primehub jobs submit < /home/jovyan/artifacts/README" 118 | } 119 | EOF 120 | ``` 121 | 122 | Using `list-artifacts` and `download-artifacts` get check and get files: 123 | 124 | ``` 125 | $ primehub jobs list-artifacts job-202108180759-z2legx 126 | name size lastModified 127 | ------ ------ ------------------------ 128 | README 8 2021-08-18T07:59:19.642Z 129 | ``` 130 | 131 | Saving the README file to the current directory `.` 132 | 133 | ``` 134 | $ primehub jobs download-artifacts job-202108180759-z2legx README . 135 | ``` 136 | ``` 137 | $ cat README 138 | show me 139 | ``` 140 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/me.md: -------------------------------------------------------------------------------- 1 | `me` command shows the account information. It is useful when you have no idea with your api-token belongs to which 2 | user. 3 | 4 | ``` 5 | $ primehub me 6 | id: a7db12dc-04fa-419c-9cd7-af768575a871 7 | username: phadmin 8 | firstName: None 9 | lastName: None 10 | email: dev+phadmin@infuseai.io 11 | isAdmin: True 12 | ``` 13 | 14 | ### Notes 15 | 16 | The two commands are the same, because the `me` command group only has one command `me`, it could use shortcut form: 17 | 18 | ``` 19 | # shortcut form 20 | primehub me 21 | ``` 22 | 23 | ``` 24 | # 25 | primehub me me 26 | ``` 27 | 28 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/notebooks.md: -------------------------------------------------------------------------------- 1 | See log messages from the user's notebook 2 | 3 | ``` 4 | primehub notebooks logs 5 | ``` 6 | 7 | ``` 8 | Set username to: jovyan 9 | usermod: no changes 10 | Changing ownership of /datasets/kaggle to 1000:100 with options '' 11 | Changing ownership of /phfs to 1000:100 with options '' 12 | Changing ownership of /home/jovyan to 1000:100 with options '' 13 | Granting jovyan sudo access and appending /opt/conda/bin to sudo PATH 14 | Executing the command: jupyter labhub --ip=0.0.0.0 --port=8888 --NotebookApp.default_url=/lab 15 | [W 2021-08-03 10:01:20.065 SingleUserLabApp configurable:190] Config option `open_browser` not recognized by `SingleUserLabApp`. Did you mean one of: `browser, expose_app_in_browser`? 16 | [WARNING 2021-08-03 10:01:21.567 SingleUserLabApp zmqhandlers:275] Couldn't authenticate WebSocket connection 17 | [WARNING 2021-08-03 10:01:21.601 SingleUserLabApp log:181] 403 GET /user/phadmin/api/kernels/cc295b85-35dc-4c6b-b8fd-ac3885224738/channels?session_id=be318bb4-b241-4b57-83a3-63f10b746ba8 (@36.225.16.122) 35.68ms 18 | ``` 19 | 20 | ### Notes 21 | 22 | If the notebook has not been ready, you might get status message: 23 | 24 | ``` 25 | {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"container \"notebook\" in pod \"jupyter-phadmin\" is waiting to start: PodInitializing","reason":"BadRequest","code":400} 26 | ``` 27 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/recurring-jobs.md: -------------------------------------------------------------------------------- 1 | ### Recurring Jobs 2 | [Recurring jobs](https://docs.primehub.io/docs/job-scheduling-feature) 3 | is a feature to make you create a job which is not running immediately. 4 | 5 | ### Fields for creating 6 | 7 | | field | required | type | description | 8 | | --- | --- | --- | --- | 9 | | displayName | required | string | display name | 10 | | instanceType | required | string | instance type which allocates resources for the job | 11 | | image | required | string | image which the job run bases on | 12 | | command | required | string | sequential commands of the job context | 13 | | recurrence | required | object | rule of trigger recurrence | 14 | | activeDeadlineSeconds | optional | int | a running job will be cancelled after this time period (in seconds) | 15 | 16 | #### Recurrence options 17 | We can select one of presets of rules or customize a rule based on [Cron](https://en.wikipedia.org/wiki/Cron) syntax. 18 | e.g., `0 4 * * *` represents 4 AM every day. 19 | 20 | | Options | Description | 21 | | --- | --- | 22 | | on-demand | trigger jobs by user request | 23 | | daily | a preset; trigger a job at 4 AM everyday | 24 | | weekly | a preset; trigger a job at 4 AM on Sunday every week | 25 | | monthly | a preset; trigger a job at 4 AM on 1st every month | 26 | | custom | customize the rule of the trigger recurrence | 27 | 28 | Only with "custom' type, we need to specify corresponding cron column. 29 | 30 | Format examples: 31 | ``` 32 | "recurrence": {"type": "daily"} 33 | "recurrence": {"type": "on-demand", "cron": ""} 34 | "recurrence": {"type": "custom", "cron": "45 23 * * 6"} # at 23:45 (11:45 PM) every Saturday 35 | ``` 36 | 37 | ### Create a recurring job 38 | 39 | Here is an example copied from [Jobs](jobs.md): 40 | 41 | ``` 42 | primehub jobs submit <` 102 | 103 | ``` 104 | $ primehub jobs submit --from recurrence-tdcfta 105 | ``` 106 | ``` 107 | job: 108 | id: job-202108180808-8tnk6z 109 | ``` 110 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/secrets.md: -------------------------------------------------------------------------------- 1 | ### Query secrets 2 | 3 | `secrets` command is used for querying secrets. You will use `id` with features that support **image pull secret**, such as the `Deployments`. 4 | 5 | Find all secrets with `list` command: 6 | 7 | ``` 8 | $ primehub secrets list 9 | 10 | id name type 11 | -------------- -------- ---------- 12 | image-example1 example1 kubernetes 13 | ``` 14 | 15 | Get a secret with `get` command: 16 | 17 | ``` 18 | $ primehub secrets get image-example1 19 | 20 | id: image-example1 21 | name: example1 22 | type: kubernetes 23 | ``` 24 | -------------------------------------------------------------------------------- /primehub/extras/templates/examples/volumes.md: -------------------------------------------------------------------------------- 1 | The `volumes` command is a group specific resource. It only works after the `group` assigned. 2 | 3 | Using `list` to find all volumes in your group: 4 | 5 | ``` 6 | $ primehub volumes list 7 | ``` 8 | 9 | ``` 10 | id name displayName description type 11 | ------ ------ ------------- ------------- ------ 12 | kaggle kaggle kaggle pv 13 | ``` 14 | 15 | If you already know the name of a volume, use the `get` to get a single entry: 16 | 17 | ``` 18 | $ primehub volumes get kaggle 19 | ``` 20 | 21 | ``` 22 | primehub volumes get kaggle 23 | id: kaggle 24 | name: kaggle 25 | displayName: kaggle 26 | description: 27 | type: pv 28 | ``` 29 | -------------------------------------------------------------------------------- /primehub/images.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Optional 3 | 4 | from primehub import Helpful, cmd, Module, primehub_load_config 5 | from primehub.utils import PrimeHubException 6 | from primehub.utils.optionals import file_flag 7 | from primehub.resource_operations import GroupResourceOperation 8 | from primehub.utils.validator import validate_name 9 | 10 | 11 | class Images(Helpful, Module, GroupResourceOperation): 12 | """ 13 | List images or get an image from the list 14 | """ 15 | resource_name = 'images' 16 | query = """ 17 | { 18 | me { 19 | effectiveGroups { 20 | name 21 | images { 22 | id 23 | name 24 | displayName 25 | description 26 | useImagePullSecret 27 | spec 28 | } 29 | } 30 | } 31 | } 32 | """ 33 | 34 | @cmd(name='list', description='List images') 35 | def list(self) -> list: 36 | """ 37 | List images 38 | 39 | :rtype: list 40 | :returns: all images in the current group 41 | """ 42 | return self.do_list(Images.query, Images.resource_name) 43 | 44 | @cmd(name='get', description='Get an image by name', return_required=True) 45 | def get(self, name) -> Optional[dict]: 46 | """ 47 | Get an image from the current group 48 | 49 | :type name: str 50 | :param name: the name of an image 51 | 52 | :rtype: Optional[dict] 53 | :returns: an image 54 | """ 55 | return self.do_get(Images.query, Images.resource_name, name) 56 | 57 | @cmd(name='create', description='Create an image', optionals=[('file', file_flag)]) 58 | def _create_cmd(self, **kwargs): 59 | """ 60 | Create an image for the current group 61 | 62 | :type file: str 63 | :param file: The file path of the configurations 64 | 65 | :rtype dict 66 | :return The image 67 | """ 68 | config = primehub_load_config(filename=kwargs.get('file', None)) 69 | if not config: 70 | invalid_config('The configuration is required.') 71 | 72 | return self.create(config) 73 | 74 | def create(self, config): 75 | """ 76 | Create an image for the current group 77 | 78 | :type config: dict 79 | :param config: The configurations for creating an image 80 | 81 | :rtype dict 82 | :return The image 83 | """ 84 | payload = validate(config) 85 | payload['groups'] = {'connect': [{'id': self.group_id}]} 86 | payload['groupName'] = self.group_name 87 | 88 | query = """ 89 | mutation CreateImageMutation($data: ImageCreateInput!) { 90 | createImage(data: $data) { 91 | id 92 | } 93 | } 94 | """ 95 | 96 | results = self.request({'data': payload}, query) 97 | if 'data' not in results: 98 | return results 99 | return results['data']['createImage'] 100 | 101 | @cmd(name='delete', description='Delete an image by name', return_required=True) 102 | def delete(self, name): 103 | """ 104 | Delete an image by id 105 | 106 | :type id: str 107 | :param id: the id of an image 108 | 109 | :rtype dict 110 | :return an image 111 | """ 112 | 113 | query = """ 114 | mutation DeleteImageMutation($where: ImageWhereUniqueInput!) { 115 | deleteImage(where: $where) { 116 | id 117 | } 118 | } 119 | """ 120 | self.get(name) 121 | 122 | results = self.request({'where': {'id': name}}, query) 123 | if 'data' not in results: 124 | return results 125 | return results['data']['deleteImage'] 126 | 127 | def help_description(self): 128 | return "Get a image or list images" 129 | 130 | 131 | def validate(payload: dict, for_update=False): 132 | if not for_update: 133 | validate_name(payload) 134 | 135 | image_type = payload.get('type') 136 | if image_type is not None and image_type not in ['cpu', 'gpu', 'both']: 137 | raise PrimeHubException("type should be one of ['cpu', 'gpu', 'both']") 138 | url = payload.get('url') 139 | if url is None: 140 | raise PrimeHubException("url is required") 141 | 142 | return payload 143 | 144 | 145 | def invalid_config(message: str): 146 | example = """ 147 | { 148 | "name": "base", 149 | "displayName": "Base image", 150 | "description": "base-notebook with python 3.7", 151 | "type": "both", 152 | "url": "infuseai/docker-stacks:base-notebook-63fdf50a", 153 | "urlForGpu": "infuseai/docker-stacks:base-notebook-63fdf50a-gpu" 154 | } 155 | """.strip() 156 | raise PrimeHubException(message + "\n\nExample:\n" + json.dumps(json.loads(example), indent=2)) 157 | -------------------------------------------------------------------------------- /primehub/info.py: -------------------------------------------------------------------------------- 1 | from primehub import Helpful, cmd, Module 2 | 3 | 4 | class CliInformation(Helpful, Module): 5 | 6 | @cmd(name='info', description='Show PrimeHub Cli information') 7 | def info(self): 8 | me = self.primehub.me.me() 9 | me['user_id'] = me['id'] 10 | 11 | current_group = self.primehub.groups.get(self.group_name) 12 | images = [x['name'] for x in self.primehub.images.list()] 13 | instance_types = [x['name'] for x in self.primehub.instancetypes.list()] 14 | volumes = [x['name'] for x in self.primehub.volumes.list()] 15 | 16 | if not current_group: 17 | group_status = " (No matched group for name %s)" % self.group_name 18 | else: 19 | group_status = """ 20 | Id: %(id)s 21 | Name: %(name)s 22 | Display Name: %(displayName)s 23 | Group Quota: 24 | CPU: %(projectQuotaCpu)s 25 | GPU: %(projectQuotaGpu)s 26 | Memory: %(projectQuotaMemory)s 27 | User Quota: 28 | CPU: %(quotaCpu)s 29 | GPU: %(quotaGpu)s 30 | Memory: %(quotaMemory)s 31 | """ % current_group 32 | 33 | def indent2(lines): 34 | return "\n ".join(lines) 35 | 36 | args = dict(endpoint=self.endpoint, group_status=group_status.strip(), 37 | images=indent2(images), instance_types=indent2(instance_types), volumes=indent2(volumes)) 38 | output = """Endpoint: %(endpoint)s 39 | User: 40 | Id: %(user_id)s 41 | Username: %(username)s 42 | Email: %(email)s 43 | First Name: %(firstName)s 44 | Last Name: %(lastName)s 45 | Is Admin: %(isAdmin)s 46 | Current Group: 47 | %(group_status)s 48 | Images: 49 | %(images)s 50 | InstanceTypes: 51 | %(instance_types)s 52 | Volumes: 53 | %(volumes)s 54 | """ % ({**me, **args}) 55 | return output 56 | 57 | def help_description(self): 58 | return "Display the user information and the selected group information" 59 | -------------------------------------------------------------------------------- /primehub/instancetypes.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from primehub import Helpful, cmd, Module 4 | 5 | 6 | class InstanceTypes(Helpful, Module): 7 | """ 8 | List instance types or get an instance type from the list 9 | """ 10 | 11 | def _list_instance_types(self, query): 12 | results = self.request({}, query) 13 | for g in results['data']['me']['effectiveGroups']: 14 | if self.group_name == g['name']: 15 | return g['instanceTypes'] 16 | return [] 17 | 18 | @cmd(name='list', description='List instance types') 19 | def list(self) -> list: 20 | """ 21 | List instance types 22 | 23 | :rtype: list 24 | :returns: all instance types in the current group 25 | """ 26 | 27 | query = """ 28 | query { 29 | me { 30 | effectiveGroups { 31 | name 32 | instanceTypes { 33 | name 34 | displayName 35 | description 36 | cpuRequest 37 | cpuLimit 38 | memoryRequest 39 | memoryLimit 40 | gpuLimit 41 | global 42 | } 43 | } 44 | } 45 | } 46 | """ 47 | 48 | return self._list_instance_types(query) 49 | 50 | @cmd(name='get', description='Get an instance type by name', return_required=True) 51 | def get(self, name) -> Optional[dict]: 52 | """ 53 | Get an instance type from the current group 54 | 55 | :type name: str 56 | :param name: the name of an instance type 57 | 58 | :rtype: Optional[dict] 59 | :returns: an instance type 60 | """ 61 | query = """ 62 | query { 63 | me { 64 | effectiveGroups { 65 | name 66 | instanceTypes { 67 | name 68 | displayName 69 | description 70 | cpuRequest 71 | cpuLimit 72 | memoryRequest 73 | memoryLimit 74 | gpuLimit 75 | global 76 | tolerations { 77 | operator 78 | key 79 | value 80 | effect 81 | } 82 | nodeSelector 83 | } 84 | } 85 | } 86 | } 87 | """ 88 | results = self._list_instance_types(query) 89 | for x in results: 90 | if x['name'] == name: 91 | return x 92 | 93 | return None 94 | 95 | def help_description(self): 96 | return "Get an instance types of list instance types" 97 | -------------------------------------------------------------------------------- /primehub/me.py: -------------------------------------------------------------------------------- 1 | from primehub import Helpful, cmd, Module 2 | 3 | 4 | class Me(Helpful, Module): 5 | 6 | @cmd(name='me', description='Get user information', return_required=True) 7 | def me(self) -> dict: 8 | """ 9 | Get account information 10 | 11 | :rtype: dict 12 | :returns: account information 13 | """ 14 | query = """ 15 | query { 16 | me { 17 | id 18 | username 19 | firstName 20 | lastName 21 | email 22 | isAdmin 23 | } 24 | } 25 | """ 26 | result = self.request({}, query) 27 | if 'data' in result and 'me' in result['data']: 28 | return result['data']['me'] 29 | return result 30 | 31 | def help_description(self): 32 | return "Show user account" 33 | -------------------------------------------------------------------------------- /primehub/notebooks.py: -------------------------------------------------------------------------------- 1 | from typing import Iterator 2 | 3 | from primehub import Helpful, cmd, Module 4 | import urllib.parse 5 | 6 | from primehub.utils.optionals import toggle_flag 7 | 8 | 9 | class Notebooks(Helpful, Module): 10 | """ 11 | The notebooks module provides functions to manage Primehub Notebooks 12 | """ 13 | 14 | @cmd(name='logs', description='Get notebooks logs', optionals=[('follow', toggle_flag), ('tail', int)]) 15 | def logs(self, **kwargs) -> Iterator[str]: 16 | """ 17 | Get notebooks logs 18 | 19 | :type follow: bool 20 | :param follow: Wait for additional logs to be appended 21 | 22 | :type tail: int 23 | :param tail: Show last n lines 24 | 25 | :rtype str 26 | :return The notebooks logs 27 | """ 28 | follow = kwargs.get('follow', False) 29 | tail = kwargs.get('tail', 10) 30 | 31 | endpoint = urllib.parse.urljoin(self.endpoint, '/api/logs/jupyterhub') 32 | return self.primehub.request_logs(endpoint, follow, tail) 33 | 34 | def help_description(self): 35 | return "Get notebooks logs" 36 | -------------------------------------------------------------------------------- /primehub/resource_operations.py: -------------------------------------------------------------------------------- 1 | from primehub.utils import resource_not_found 2 | 3 | 4 | class GroupResourceOperation(object): 5 | 6 | def do_list(self, query, resource_key): 7 | current_group = self.group_name 8 | 9 | results = self.request({}, query) 10 | for g in results['data']['me']['effectiveGroups']: 11 | if current_group == g['name']: 12 | return g[resource_key] 13 | return [] 14 | 15 | def do_get(self, query, resource_key, resource_name): 16 | resources = self.do_list(query, resource_key) 17 | data = [x for x in resources if x['name'] == resource_name] 18 | if data: 19 | return data[0] 20 | resource_not_found(resource_key, resource_name, 'name') 21 | -------------------------------------------------------------------------------- /primehub/secrets.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from primehub import Helpful, cmd, Module 4 | 5 | 6 | class Secrets(Helpful, Module): 7 | """ 8 | Query secrets for Image Pull Secrets 9 | """ 10 | 11 | @cmd(name='list', description='List secrets') 12 | def list(self) -> list: 13 | """ 14 | List secrets 15 | 16 | :rtype: list 17 | :returns: secrets 18 | """ 19 | query = """ 20 | { 21 | secrets(where: { ifDockerConfigJson: true }) { 22 | id 23 | name 24 | type 25 | } 26 | } 27 | """ 28 | results = self.request({}, query) 29 | return results['data']['secrets'] 30 | 31 | @cmd(name='get', description='Get a secret by id', return_required=True) 32 | def get(self, id: str) -> Optional[dict]: 33 | """ 34 | Get the secret by id 35 | 36 | :type id: str 37 | :param id: the id of a secret 38 | 39 | :rtype: Optional[dict] 40 | :returns: a secret 41 | """ 42 | secret = self.list() 43 | s = [x for x in secret if x['id'] == id] 44 | if s: 45 | return s[0] 46 | return None 47 | 48 | def help_description(self): 49 | return "Get a secret or list secrets" 50 | -------------------------------------------------------------------------------- /primehub/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | 6 | class PrimeHubException(BaseException): 7 | pass 8 | 9 | 10 | class GraphQLException(PrimeHubException): 11 | pass 12 | 13 | 14 | class GroupIsRequiredException(PrimeHubException): 15 | pass 16 | 17 | 18 | class UserRejectAction(PrimeHubException): 19 | pass 20 | 21 | 22 | class RequestException(PrimeHubException): 23 | pass 24 | 25 | 26 | class ResponseException(PrimeHubException): 27 | pass 28 | 29 | 30 | class PrimeHubReturnsRequiredException(PrimeHubException): 31 | pass 32 | 33 | 34 | class ResourceNotFoundException(PrimeHubException): 35 | pass 36 | 37 | 38 | class SharedFileException(PrimeHubException): 39 | pass 40 | 41 | 42 | class DatasetsException(PrimeHubException): 43 | pass 44 | 45 | 46 | def reject_action(action): 47 | raise UserRejectAction( 48 | 'User rejects action [%s], please use the flag "--yes-i-really-mean-it" to allow the action.' % action) 49 | 50 | 51 | def group_required(): 52 | raise GroupIsRequiredException('No group information, please configure the active group first.') 53 | 54 | 55 | def group_not_found(group): 56 | raise GroupIsRequiredException('No group information for [%s], please check the configuration again.' % group) 57 | 58 | 59 | def resource_not_found(resource_type: str, key: str, key_type: str): 60 | raise ResourceNotFoundException(resource_type, key, key_type) 61 | 62 | 63 | def create_logger(name) -> logging.Logger: 64 | log_level = logging.WARNING 65 | if os.environ.get('PRIMEHUB_SDK_LOG_LEVEL') == 'DEBUG': 66 | log_level = logging.DEBUG 67 | if os.environ.get('PRIMEHUB_SDK_LOG_LEVEL') == 'INFO': 68 | log_level = logging.INFO 69 | 70 | log = logging.getLogger(name) 71 | log.setLevel(log_level) 72 | handler = logging.StreamHandler(sys.stderr) 73 | handler.setLevel(log_level) 74 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 75 | handler.setFormatter(formatter) 76 | log.addHandler(handler) 77 | return log 78 | -------------------------------------------------------------------------------- /primehub/utils/completion.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ..cli import create_sdk 3 | from .decorators import find_actions 4 | 5 | variables = ('COMP_CWORD', 'COMP_LINE', 'COMP_POINT', 'COMP_WORDS', 'cur', 'prev') 6 | 7 | sdk = create_sdk() 8 | 9 | PRIMEHUB_AUTO_COMPLETION_LOG = (os.environ.get('PRIMEHUB_AUTO_COMPLETION_LOG', 'false') == 'true') 10 | 11 | 12 | def _debug_log(x): 13 | if not PRIMEHUB_AUTO_COMPLETION_LOG: 14 | return 15 | with open('.auto-primehub.txt', 'a') as fh: 16 | fh.write(x) 17 | fh.write('\n') 18 | 19 | 20 | def _debug_variables(): 21 | if not PRIMEHUB_AUTO_COMPLETION_LOG: 22 | return 23 | with open('.auto-primehub.txt', 'a') as fh: 24 | fh.write('-' * 16) 25 | fh.write('\n') 26 | for v in variables: 27 | fh.write(f'{v}={os.environ.get(v)}\n') 28 | 29 | 30 | def auto_complete(): 31 | _debug_variables() 32 | 33 | # first level command groups 34 | # primehub 35 | # cur = os.environ.get('cur') 36 | prev = os.environ.get('prev') 37 | current_index = os.environ.get('COMP_CWORD') 38 | command_line = os.environ.get('COMP_LINE') 39 | if prev == 'primehub': 40 | for k in sdk.commands: 41 | if k.startswith(':'): 42 | continue 43 | print(k) 44 | return 45 | 46 | # primehub 47 | if current_index == '2': 48 | _debug_log(f'{prev} => {sdk.commands.keys()}') 49 | if prev != 'admin' and prev in sdk.commands: 50 | for verb in find_actions(sdk.commands[prev]): 51 | print(verb['name']) 52 | elif prev == 'admin': 53 | for k in sdk.admin_commands: 54 | if k.startswith(':'): 55 | continue 56 | print(k) 57 | 58 | if current_index == '3' and command_line.startswith('primehub admin '): 59 | if prev in sdk.admin_commands: 60 | for verb in find_actions(sdk.admin_commands[prev]): 61 | print(verb['name']) 62 | 63 | 64 | if __name__ == '__main__': 65 | auto_complete() 66 | -------------------------------------------------------------------------------- /primehub/utils/core.py: -------------------------------------------------------------------------------- 1 | import random 2 | import re 3 | from collections import UserDict 4 | 5 | 6 | class CommandContainer(UserDict): 7 | 8 | def keys(self): 9 | return [x for x in super(CommandContainer, self).keys() if not x.startswith(':')] 10 | 11 | def items(self): 12 | return [(k, v) for (k, v) in super(CommandContainer, self).items() if not k.startswith(':')] 13 | 14 | def __getitem__(self, item): 15 | try: 16 | return super(CommandContainer, self).__getitem__(item) 17 | except KeyError: 18 | return super(CommandContainer, self).__getitem__(f':{item}') 19 | 20 | def __contains__(self, item): 21 | has_item = super(CommandContainer, self).__contains__(item) 22 | if has_item: 23 | return True 24 | return super(CommandContainer, self).__contains__(f':{item}') 25 | 26 | 27 | def auto_gen_id(name: str): 28 | normalized_name = re.sub(r'[\W_]', '-', name).lower() 29 | random_string = str(float.hex(random.random()))[4:9] 30 | return f'{normalized_name}-{random_string}' 31 | -------------------------------------------------------------------------------- /primehub/utils/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from inspect import signature 3 | from types import FunctionType 4 | from typing import Dict 5 | 6 | from primehub.utils import create_logger 7 | from primehub.utils.optionals import default_optional_builder 8 | 9 | logger = create_logger('decorator') 10 | 11 | __command_groups__: Dict[str, list] = dict() 12 | __actions__: Dict[str, dict] = dict() 13 | __requires_permission__: Dict[str, str] = dict() 14 | 15 | 16 | def find_actions(sub_command): 17 | m = sub_command.__module__ 18 | if m in __command_groups__: 19 | actions = __command_groups__[m] 20 | from operator import itemgetter 21 | return sorted(actions, key=itemgetter('name')) 22 | return [] 23 | 24 | 25 | def _a(module, action): 26 | return "{}.{}".format(module.__module__, action) 27 | 28 | 29 | def find_action_info(module, action): 30 | logger.debug("find_action_info (%s, %s)", module, action) 31 | if _a(module, action) in __actions__: 32 | return __actions__[_a(module, action)] 33 | 34 | 35 | def find_action_method(module, action): 36 | info = find_action_info(module, action) 37 | if info: 38 | return info['func'] 39 | 40 | 41 | def register_to_command_group(func, cmd_args): 42 | """ 43 | A python module is a .py file, there are one or more commands in a module. 44 | A module is a command group with lots of actions and each action is a method in the module. 45 | 46 | For example: 47 | module cli_config could have {set_endpoint, set_token_, set_group} actions. 48 | 49 | We keep: 50 | * an action with the key "." for O(1) action search. 51 | * an module with the key "" for O(1) module-actions search 52 | """ 53 | 54 | if func.__module__ not in __command_groups__: 55 | __command_groups__[func.__module__] = [] 56 | 57 | __command_groups__[func.__module__].append(cmd_args) 58 | __actions__["{}.{}".format(func.__module__, cmd_args['name'])] = cmd_args 59 | 60 | 61 | def cmd(**cmd_args): 62 | if 'name' not in cmd_args: 63 | raise ValueError('name field is required') 64 | if 'description' not in cmd_args: 65 | raise ValueError('name description is required') 66 | if 'optionals' not in cmd_args: 67 | cmd_args['optionals'] = [] 68 | if 'return_required' not in cmd_args: 69 | cmd_args['return_required'] = False 70 | 71 | for idx, opts in enumerate(cmd_args['optionals']): 72 | maybe_builder = opts[-1] 73 | if not isinstance(maybe_builder, FunctionType): 74 | cmd_args['optionals'][idx] = tuple(list(opts) + [default_optional_builder]) 75 | 76 | def inner(func): 77 | make_command_references(func) 78 | 79 | @wraps(func) 80 | def wrapper(*args, **kwargs): 81 | return func(*args, **kwargs) 82 | 83 | return wrapper 84 | 85 | def make_command_references(func): 86 | # TODO only generate references when invoked from primehub-cli 87 | cmd_args['module'] = func.__module__ 88 | cmd_args['func'] = func.__name__ 89 | sig = signature(func) 90 | import inspect 91 | 92 | def info(x): 93 | if x.annotation is inspect.Parameter.empty: 94 | t = str 95 | else: 96 | t = x.annotation 97 | return x.name, t, str(x.kind) == 'VAR_KEYWORD', 98 | 99 | cmd_args['arguments'] = [info(x) for x in sig.parameters.values() if x.name != 'self'] 100 | logger.debug("cmd -> %s", cmd_args) 101 | register_to_command_group(func, cmd_args) 102 | 103 | return inner 104 | 105 | 106 | def show_debug_info(): 107 | print("Command Groups:") 108 | for k, v in __command_groups__.items(): 109 | print(" {}".format(k)) 110 | for a in v: 111 | print(" {}".format(a)) 112 | print("") 113 | print("Actions:") 114 | for k, v in __actions__.items(): 115 | print(" {}\n {}".format(k, v)) 116 | print("") 117 | print("Permission:") 118 | for k, v in __requires_permission__.items(): 119 | print(" {}".format(k)) 120 | print("") 121 | 122 | 123 | class Command(object): 124 | 125 | def __init__(self, name, description, optional_arguments: list): 126 | self.name = name 127 | self.description = description 128 | self.optional_arguments = optional_arguments 129 | -------------------------------------------------------------------------------- /primehub/utils/http_client.py: -------------------------------------------------------------------------------- 1 | import json 2 | from json import JSONDecodeError 3 | from typing import Iterator, Callable, Optional 4 | 5 | import requests # type: ignore 6 | 7 | from primehub.utils import ResponseException, RequestException, GraphQLException, create_logger, \ 8 | ResourceNotFoundException 9 | 10 | logger = create_logger('http') 11 | 12 | 13 | class Client(object): 14 | 15 | def __init__(self, primehub_config): 16 | self.primehub_config = primehub_config 17 | self.timeout = 10 18 | 19 | def request(self, variables: dict, query: str, error_handler: Optional[Callable] = None): 20 | request_body = dict(variables=json.dumps(variables), query=query) 21 | logger.debug('request body: {}'.format(request_body)) 22 | headers = {'authorization': 'Bearer {}'.format(self.primehub_config.api_token)} 23 | try: 24 | content = requests.post(self.primehub_config.endpoint, data=request_body, headers=headers, 25 | timeout=self.timeout).text 26 | logger.debug('response: {}'.format(content)) 27 | result = json.loads(content) 28 | if 'errors' in result: 29 | if error_handler: 30 | error_handler(result) 31 | raise GraphQLException(result) 32 | return result 33 | except JSONDecodeError: 34 | raise ResponseException("Response is not valid JSON:\n{}".format(content)) 35 | except ResourceNotFoundException as e: 36 | raise e 37 | except GraphQLException as e: 38 | raise e 39 | except BaseException as e: 40 | raise RequestException(e) 41 | 42 | def request_logs(self, endpoint, follow, tail) -> Iterator[bytes]: 43 | params = {'follow': 'false'} 44 | if follow: 45 | params['follow'] = 'true' 46 | if tail: 47 | params['tailLines'] = str(tail) 48 | headers = {'authorization': 'Bearer {}'.format(self.primehub_config.api_token)} 49 | 50 | with requests.get(endpoint, headers=headers, params=params, stream=follow) as response: 51 | for chunk in response.iter_content(chunk_size=8192): 52 | yield chunk 53 | 54 | def request_file(self, endpoint, dest): 55 | headers = {'authorization': 'Bearer {}'.format(self.primehub_config.api_token)} 56 | with requests.get(endpoint, headers=headers) as r: 57 | with open(dest, 'wb') as f: 58 | f.write(r.content) 59 | return 60 | 61 | def upload_file(self, endpoint, src): 62 | headers = {'authorization': 'Bearer {}'.format(self.primehub_config.api_token)} 63 | with open(src, 'rb') as f: 64 | r = requests.post(endpoint, headers=headers, data=f) 65 | return r.json() 66 | 67 | 68 | if __name__ == '__main__': 69 | print(Client.__module__) 70 | -------------------------------------------------------------------------------- /primehub/utils/optionals.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | 4 | def default_optional_builder(parser: ArgumentParser, name: str, type_of_arg: type): 5 | parser.add_argument("--" + name, type=type_of_arg) 6 | 7 | 8 | def toggle_flag(parser: ArgumentParser, name: str): 9 | parser.add_argument("--" + name, action="store_true", default=False) 10 | 11 | 12 | def file_flag(parser: ArgumentParser, name: str): 13 | if name != 'file': 14 | raise ValueError(f'name should be file, but it is {name}') 15 | parser.add_argument("--file", "-f", dest='file') 16 | -------------------------------------------------------------------------------- /primehub/utils/permission.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | from primehub.utils import reject_action 4 | from primehub.utils.decorators import __requires_permission__, logger, __command_groups__ 5 | 6 | 7 | def has_permission_flag(cmd_args: dict): 8 | k = "{}.{}".format(cmd_args['module'], cmd_args['func']) 9 | if k in __requires_permission__: 10 | return True 11 | return False 12 | 13 | 14 | def disable_reject_action(action): 15 | logger.debug('@ask_for_permission is disable, the action will pass %s', action) 16 | 17 | 18 | reject_action_function = disable_reject_action 19 | 20 | 21 | def enable_ask_for_permission_feature(): 22 | global reject_action_function 23 | reject_action_function = reject_action 24 | 25 | 26 | def ask_for_permission(func): 27 | k = "{}.{}".format(func.__module__, func.__name__) 28 | __requires_permission__[k] = "" 29 | 30 | @wraps(func) 31 | def wrapper(*args, **kwargs): 32 | if not kwargs.get('--yes-i-really-mean-it', False): 33 | cmd_args = None 34 | try: 35 | actions = __command_groups__[func.__module__] 36 | cmd_args = [x for x in actions if x['func'] == func.__name__] 37 | except BaseException: 38 | pass 39 | if cmd_args: 40 | reject_action_function(cmd_args[0]['name']) 41 | else: 42 | reject_action_function(func.__name__) 43 | 44 | return func(*args, **kwargs) 45 | 46 | return wrapper 47 | -------------------------------------------------------------------------------- /primehub/version.py: -------------------------------------------------------------------------------- 1 | from primehub import Helpful, cmd, Module, __version__ 2 | 3 | 4 | class Version(Helpful, Module): 5 | 6 | @cmd(name='version', description='show version number', return_required=True) 7 | def version(self) -> str: 8 | return __version__ 9 | 10 | def help_description(self): 11 | return "Display the version of PrimeHub Python SDK" 12 | -------------------------------------------------------------------------------- /primehub/volumes.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from primehub import Helpful, cmd, Module 4 | from primehub.resource_operations import GroupResourceOperation 5 | 6 | 7 | class Volumes(Helpful, Module, GroupResourceOperation): 8 | """ 9 | List volumes or get a volume entry from the list 10 | """ 11 | resource_name = 'datasets' 12 | query = """ 13 | query { 14 | me { 15 | effectiveGroups { 16 | name 17 | datasets { 18 | id 19 | name 20 | displayName 21 | description 22 | type 23 | } 24 | } 25 | } 26 | } 27 | """ 28 | 29 | @cmd(name='list', description='List volumes') 30 | def list(self) -> list: 31 | """ 32 | List volumes 33 | 34 | :rtype: list 35 | :returns: all volumes in the current group 36 | """ 37 | return self.do_list(Volumes.query, Volumes.resource_name) 38 | 39 | @cmd(name='get', description='Get a volume by name', return_required=True) 40 | def get(self, name) -> Optional[dict]: 41 | """ 42 | Get a volume from the current group 43 | :type name: str 44 | :param name: the name of a volume 45 | :rtype: Optional[dict] 46 | :returns: a volume 47 | """ 48 | return self.do_get(Volumes.query, Volumes.resource_name, name) 49 | 50 | def help_description(self): 51 | return "Get a volume or list volumes" 52 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # install dependencies from setup.py (editable mode) 2 | -e . -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | flake8-max-line-length = 120 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | from distutils.core import setup 4 | 5 | from setuptools import find_packages # type: ignore 6 | 7 | 8 | def _get_version(): 9 | version_file = os.path.normpath(os.path.join(os.path.dirname(__file__), 'primehub', 'VERSION')) 10 | with open(version_file) as fh: 11 | version = fh.read().strip() 12 | return version 13 | 14 | 15 | setup(name='primehub-python-sdk', 16 | version=_get_version(), 17 | description='PrimeHub Python SDK', 18 | long_description=open('README.md').read(), 19 | long_description_content_type='text/markdown', 20 | author='qrtt1', 21 | author_email='qrtt1@infuseai.io', 22 | url='https://github.com/InfuseAI/primehub-python-sdk', 23 | entry_points={ 24 | 'console_scripts': ['primehub = primehub.cli:main', 'doc-primehub = primehub.extras.doc_generator:main', 25 | 'auto-primehub = primehub.utils.completion:auto_complete'] 26 | }, 27 | python_requires=">=3.6", 28 | packages=find_packages(), 29 | install_requires=['requests', 'tabulate==0.8.9', 'types-tabulate==0.8.2', 'types-requests'], 30 | extras_require={ 31 | 'dev': [ 32 | 'pytest>=4.6', 33 | 'pytest-flake8', 34 | 'flake8==3.9.2', 35 | 'pytest-mypy', 36 | 'pytest-cov', 37 | 'Jinja2', 'types-Jinja2', 38 | 'twine', 39 | 'importlib-metadata<5' 40 | ], 41 | }, 42 | project_urls={ 43 | "Bug Tracker": "https://github.com/InfuseAI/primehub/issues", 44 | }, 45 | classifiers=[ 46 | "Programming Language :: Python :: 3", 47 | "License :: OSI Approved :: Apache Software License", 48 | "Operating System :: OS Independent", 49 | "Development Status :: 4 - Beta" 50 | ], 51 | package_data={ 52 | 'primehub': ['*.json', 'VERSION'] 53 | }) 54 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import io 3 | import json 4 | import os 5 | from unittest import TestCase, mock 6 | 7 | from primehub import PrimeHub, PrimeHubConfig, cli 8 | from primehub.utils import create_logger 9 | 10 | logger = create_logger('primehub-test') 11 | 12 | 13 | def reset_stderr_stdout(func): 14 | @functools.wraps(func) 15 | def wrapper(*args, **kwargs): 16 | try: 17 | return func(*args, **kwargs) 18 | finally: 19 | args[0].reset_cli_output() 20 | 21 | return wrapper 22 | 23 | 24 | class BaseTestCase(TestCase): 25 | """ 26 | A base test case which mocks out the low-level HTTP Client to prevent 27 | any actual calls to the API. 28 | """ 29 | 30 | def setUp(self) -> None: 31 | 32 | import primehub.utils.http_client 33 | 34 | primehub.utils.http_client.Client.request = mock.MagicMock() 35 | self.mock_request = primehub.utils.http_client.Client.request 36 | 37 | import primehub 38 | self.test_default_config_path = self.tempfile() 39 | primehub.PrimeHubConfig.get_default_path = mock.MagicMock() 40 | primehub.PrimeHubConfig.get_default_path.return_value = self.test_default_config_path 41 | print("the default path is ", self.test_default_config_path) 42 | 43 | self.sdk = PrimeHub(PrimeHubConfig()) 44 | self.sdk.stderr = io.StringIO() 45 | self.sdk.stdout = io.StringIO() 46 | 47 | def tearDown(self) -> None: 48 | os.environ['PRIMEHUB_GROUP'] = "" 49 | os.environ['PRIMEHUB_API_TOKEN'] = "" 50 | os.environ['PRIMEHUB_API_ENDPOINT'] = "" 51 | 52 | @reset_stderr_stdout 53 | def cli_stderr(self, argv: list): 54 | print("cli_stderr", argv) 55 | self.invoke_cli(argv) 56 | return self.sdk.stderr.getvalue() 57 | 58 | @reset_stderr_stdout 59 | def cli_stdout(self, argv: list): 60 | print("cli_stdout", argv) 61 | self.invoke_cli(argv) 62 | return self.sdk.stdout.getvalue() 63 | 64 | def invoke_cli(self, argv: list): 65 | try: 66 | import sys 67 | sys.argv = argv 68 | if [x for x in argv if not isinstance(x, str)]: 69 | raise ValueError('all arguments must be a str type') 70 | if '--json' not in sys.argv: 71 | sys.argv.append('--json') 72 | cli.main(sdk=self.sdk) 73 | except SystemExit: 74 | pass 75 | 76 | def reset_cli_output(self): 77 | logger.info("\n{} reset_cli_output {}".format("=" * 40, "=" * 40)) 78 | logger.info("\nSTDOUT:\n\n{}\n\nSTDERR:\n\n{}\n\n".format(self.sdk.stdout.getvalue(), 79 | self.sdk.stderr.getvalue())) 80 | self.sdk.stderr = io.StringIO() 81 | self.sdk.stdout = io.StringIO() 82 | 83 | def create_fake_config(self, content: dict) -> str: 84 | from tempfile import mkstemp 85 | fd, p = mkstemp('.json', text=True) 86 | with open(p, "w") as fh: 87 | fh.write(json.dumps(content)) 88 | return p 89 | 90 | def create_config_dict(self, endpoint, token, group): 91 | return {'endpoint': endpoint, 'api-token': token, 'group': dict(name=group)} 92 | 93 | def cfg_dict_with_prefix(self, prefix: str): 94 | return self.create_config_dict(prefix + ":endpoint", prefix + ":api-token", prefix + ":group") 95 | 96 | def assert_config_with_prefix(self, prefix: str, cfg: PrimeHubConfig): 97 | self.assertEqual(prefix + ':endpoint', cfg.endpoint) 98 | self.assertEqual(prefix + ':api-token', cfg.api_token) 99 | self.assertEqual(cfg.current_group.get('name'), cfg.group) 100 | self.assertEqual(prefix + ':group', cfg.current_group.get('name')) 101 | 102 | def tempfile(self): 103 | from tempfile import mkstemp 104 | fd, p = mkstemp(".data", text=True) 105 | return p 106 | 107 | def load_json_file(self, path): 108 | with open(path, "r") as fh: 109 | return json.load(fh) 110 | -------------------------------------------------------------------------------- /tests/graphql_formatter.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import tempfile 3 | 4 | 5 | def is_formatter_available(): 6 | try: 7 | process = subprocess.run(['prettier', '-h'], capture_output=True) 8 | return process.stdout.decode('utf8').startswith('Usage: prettier') 9 | except BaseException: 10 | return False 11 | 12 | 13 | def format_graphql(graphql_query): 14 | _, gql = tempfile.mkstemp(suffix=".graphql") 15 | with open(gql, 'w') as f: 16 | f.write(graphql_query) 17 | process = subprocess.run(['prettier', gql], capture_output=True) 18 | if process.returncode != 0: 19 | raise BaseException(f'Can not format the query:\n\n{graphql_query}\n') 20 | return process.stdout.decode('utf8') 21 | -------------------------------------------------------------------------------- /tests/test_admin_images.py: -------------------------------------------------------------------------------- 1 | from primehub.admin_images import validate, validate_image_type, validate_image_spec 2 | from primehub.utils import PrimeHubException 3 | from tests import BaseTestCase 4 | 5 | 6 | class TestAdminImages(BaseTestCase): 7 | 8 | def setUp(self) -> None: 9 | super(TestAdminImages, self).setUp() 10 | 11 | def check_required(self, cfg: dict, message: str, callback=None): 12 | with self.assertRaises(PrimeHubException) as context: 13 | if callback is None: 14 | validate(cfg) 15 | else: 16 | callback(cfg) 17 | 18 | self.assertTrue(isinstance(context.exception, PrimeHubException)) 19 | self.assertEqual(message, context.exception.args[0]) 20 | 21 | def test_validator(self): 22 | 23 | # check empty username 24 | self.check_required({}, 'name is required') 25 | self.check_required({'name': 'aaaa_aaa'}, 26 | "[name] should be lower case alphanumeric characters, '-' " 27 | "or '.', and must start and end with an alphanumeric character.") 28 | 29 | # check invalid type valuez 30 | self.check_required({'name': 'image-1', 'type': ''}, "type should be one of ['cpu', 'gpu', 'both']", 31 | validate_image_type) 32 | self.check_required({'name': 'image-1', 'type': 'abc'}, "type should be one of ['cpu', 'gpu', 'both']", 33 | validate_image_type) 34 | 35 | # TODO check secret useImagePullSecret after we support the "secrets" command group 36 | 37 | # check url* and imageSpec 38 | self.check_required({'name': 'image-1', 'url': 'image', 'imageSpec': { 39 | 'baseImage': 'jupyter/base-notebook', 'packages': {'pip': ['pytest']} 40 | }}, "imageSpec cannot use with url and urlForGpu") 41 | 42 | self.check_required({'name': 'image-1', 'urlForGpu': 'image-for-gpu', 'imageSpec': { 43 | 'baseImage': 'jupyter/base-notebook', 'packages': {'pip': ['pytest']} 44 | }}, "imageSpec cannot use with url and urlForGpu") 45 | 46 | # pass with valid configuration 47 | validate({'name': 'my-image.org'}) 48 | 49 | # pass with url and urlForGpu 50 | validate({'name': 'my-image', 'url': 'image-url', 'urlForGpu': 'image-gpu-url'}) 51 | 52 | # pass with imageSpec 53 | validate({'name': 'my-image', 'imageSpec': { 54 | 'baseImage': 'jupyter/base-notebook', 'packages': {'pip': ['pytest']} 55 | }}) 56 | 57 | def test_image_spec_validator(self): 58 | def s(cfg: dict): 59 | cfg['baseImage'] = 'jupyter/base-notebook' 60 | return cfg 61 | 62 | def c(payload: dict, expected: str): 63 | self.check_required(payload, expected, validate_image_spec) 64 | 65 | c({'name': 'image-1', 'imageSpec': {}}, "Invalid imageSpec: baseImage is required") 66 | c({'name': 'image-1', 'imageSpec': s({})}, "Invalid imageSpec: packages is required") 67 | 68 | c({'name': 'image-1', 'imageSpec': s({ 69 | 'packages': [] 70 | })}, "Invalid imageSpec: packages is required") 71 | 72 | c({'name': 'image-1', 'imageSpec': s({ 73 | 'packages': {} 74 | })}, "Invalid imageSpec: packages is required") 75 | 76 | c({'name': 'image-1', 'imageSpec': s({ 77 | 'packages': ['pytest']} 78 | )}, "Invalid imageSpec: packages is a dict with {apt, pip, conda} keys and you have use at least one") 79 | 80 | c({'name': 'image-1', 'imageSpec': s({ 81 | 'packages': {'abc': ['abc']} 82 | })}, "Invalid imageSpec: packages is a dict with {apt, pip, conda} keys and you have use at least one") 83 | 84 | c({'name': 'image-1', 'imageSpec': s({ 85 | 'packages': {'pip': 'pytest'} 86 | })}, "Invalid imageSpec: packages values should be a list") 87 | -------------------------------------------------------------------------------- /tests/test_admin_users.py: -------------------------------------------------------------------------------- 1 | from primehub.admin_users import validate, USERNAME_FORMAT_ERROR, EMAIL_FORMAT_ERROR 2 | from primehub.utils import PrimeHubException 3 | from tests import BaseTestCase 4 | 5 | 6 | class TestAdminUsers(BaseTestCase): 7 | 8 | def setUp(self) -> None: 9 | super(TestAdminUsers, self).setUp() 10 | 11 | def check_required(self, cfg: dict, message: str, callback=None): 12 | with self.assertRaises(PrimeHubException) as context: 13 | if callback is None: 14 | validate(cfg) 15 | else: 16 | callback(cfg) 17 | 18 | self.assertTrue(isinstance(context.exception, PrimeHubException)) 19 | self.assertEqual(message, context.exception.args[0]) 20 | 21 | def test_required_field_validator(self): 22 | 23 | # check empty username 24 | self.check_required({}, 'username is required') 25 | 26 | # checkt bad username 27 | self.check_required({'username': '-abc'}, USERNAME_FORMAT_ERROR) 28 | self.check_required({'username': 'Upper-case'}, USERNAME_FORMAT_ERROR) 29 | 30 | # pass with valid usernames 31 | validate({'username': 'user1'}) 32 | validate({'username': 'user1-a-b-d'}) 33 | validate({'username': 'user1@abc.com'}) 34 | 35 | def test_email_validator(self): 36 | 37 | payload = dict(username='user1') 38 | 39 | # pass without email 40 | validate(payload) 41 | 42 | # check bad mail format 43 | self.check_required({**payload, 'email': ''}, EMAIL_FORMAT_ERROR) 44 | self.check_required({**payload, 'email': 'email'}, EMAIL_FORMAT_ERROR) 45 | self.check_required({**payload, 'email': 'email@'}, EMAIL_FORMAT_ERROR) 46 | self.check_required({**payload, 'email': 'email@222.222.22'}, EMAIL_FORMAT_ERROR) 47 | 48 | # pass with valid email 49 | validate({**payload, 'email': 'email@example.com'}) 50 | -------------------------------------------------------------------------------- /tests/test_cli_config.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from primehub import Helpful, Module, cmd, PrimeHubConfig 4 | from tests import BaseTestCase 5 | 6 | 7 | class ConfigWatcher(Helpful, Module): 8 | 9 | @cmd(name='watch', description='watch-primehub-config') 10 | def watch(self): 11 | # ensure the config file is changed 12 | assert self.primehub.primehub_config.config_file != self.primehub.primehub_config.get_default_path() 13 | 14 | from tempfile import mkstemp 15 | fd, path = mkstemp('.json', text=True) 16 | self.primehub.primehub_config.save(path=path) 17 | 18 | return dict(config_file=path) 19 | 20 | def help_description(self): 21 | return "Big Brother Watch YOU" 22 | 23 | 24 | class TestCliConfig(BaseTestCase): 25 | 26 | def setUp(self) -> None: 27 | super(TestCliConfig, self).setUp() 28 | 29 | # clean commands, add the FakeCommand 30 | self.sdk.register_command('test_cli_config', ConfigWatcher) 31 | 32 | def test_config_options(self): 33 | """ 34 | Global Options: 35 | --config CONFIG the path of the config file 36 | --endpoint ENDPOINT the endpoint to the PrimeHub GraphQL URL 37 | --token TOKEN API Token generated from PrimeHub Console 38 | --group GROUP override the active group 39 | """ 40 | group_info = {'name': 'test-config-from-cli:group'} 41 | self.mock_request.return_value = {'data': {'me': {'effectiveGroups': [group_info]}}} 42 | c = self.make_cfg() 43 | 44 | # Verify the saved config will have test-config-from-cli:* 45 | self.assert_config_with_prefix('test-config-from-cli', c) 46 | 47 | # Verify other options 48 | group_info = {'name': 'opt:group'} 49 | self.mock_request.return_value = {'data': {'me': {'effectiveGroups': [group_info]}}} 50 | c = self.make_cfg(['--endpoint', 'opt:endpoint', '--token', 'opt:api-token', '--group', 'opt:group']) 51 | self.assert_config_with_prefix('opt', c) 52 | 53 | def make_cfg(self, extra_args: list = None): 54 | if extra_args is None: 55 | extra_args = [] 56 | 57 | cfg_path = self.create_fake_config(self.cfg_dict_with_prefix('test-config-from-cli')) 58 | output = self.cli_stdout(['app.py', 'test_cli_config', 'watch', '--config', cfg_path] + extra_args) 59 | 60 | # bug: the output buffer contains the previous result, we only take the last one 61 | output = output.strip().split("\n")[-1] 62 | 63 | saved_cfg_path = json.loads(output)['config_file'] 64 | new_cfg = PrimeHubConfig(config=saved_cfg_path) 65 | return new_cfg 66 | -------------------------------------------------------------------------------- /tests/test_cmd_config.py: -------------------------------------------------------------------------------- 1 | from primehub import PrimeHubConfig 2 | from tests import BaseTestCase 3 | 4 | 5 | class TestCmdConfig(BaseTestCase): 6 | 7 | def setUp(self) -> None: 8 | super(TestCmdConfig, self).setUp() 9 | 10 | def test_config_options(self): 11 | """ 12 | Usage: 13 | primehub config [command] 14 | 15 | Update the settings of PrimeHub SDK 16 | 17 | Available Commands: 18 | set-endpoint set endpoint and save to the config file 19 | set-token set token and save to the config file 20 | set-group set group and save to the config file 21 | """ 22 | 23 | # use cli version's config 24 | self.sdk.register_command('config', 'CliConfig') 25 | 26 | args = ['app.py', 'config', 'set-endpoint', 'config-set:endpoint'] 27 | self.cli_stdout(args) 28 | 29 | args = ['app.py', 'config', 'set-token', 'config-set:api-token'] 30 | self.cli_stdout(args) 31 | 32 | # mock the result of fetching group info 33 | group_info = { 34 | 'id': 'group-id', 35 | 'name': 'config-set:group', 36 | 'displayName': 'group-display-name' 37 | } 38 | self.mock_request.return_value = {'data': {'me': {'effectiveGroups': [group_info]}}} 39 | args = ['app.py', 'config', 'set-group', 'config-set:group'] 40 | self.cli_stdout(args) 41 | 42 | # Verify new configuration has been prefixed with 'config:set' 43 | self.assert_config_with_prefix('config-set', PrimeHubConfig()) 44 | 45 | # Verify group information updated 46 | # { 47 | # me { 48 | # effectiveGroups { 49 | # id 50 | # name 51 | # displayName 52 | # } 53 | # } 54 | # } 55 | self.assertEqual(group_info, self.load_json_file(PrimeHubConfig().get_default_path())['group']) 56 | -------------------------------------------------------------------------------- /tests/test_cmd_me.py: -------------------------------------------------------------------------------- 1 | import json 2 | from tests import BaseTestCase 3 | 4 | 5 | class TestCmdMe(BaseTestCase): 6 | """ 7 | Usage: 8 | primehub me [command] 9 | 10 | Show user account 11 | 12 | Available Commands: 13 | me Get user information 14 | """ 15 | 16 | def setUp(self) -> None: 17 | super(TestCmdMe, self).setUp() 18 | 19 | def test_me(self): 20 | # me will return anything from the query 21 | # query { 22 | # me { 23 | # id 24 | # username 25 | # firstName 26 | # lastName 27 | # email 28 | # isAdmin 29 | # } 30 | # } 31 | body = {'id': 'id', 'username': 'username', 32 | 'firstName': 'firstName', 'lastName': 'lastName', 33 | 'email': 'email', 'isAdmin': True} 34 | self.mock_request.return_value = {'data': { 35 | 'me': body}} 36 | output = self.cli_stdout(['app.py', 'me']) 37 | 38 | self.assertEqual(body, json.loads(output)) 39 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from tempfile import mkstemp 3 | 4 | from primehub import PrimeHubConfig 5 | from tests import BaseTestCase 6 | 7 | 8 | class TestPrimeHubConfig(BaseTestCase): 9 | 10 | def setUp(self) -> None: 11 | super(TestPrimeHubConfig, self).setUp() 12 | 13 | def test_primehub_config_evaluation(self): 14 | endpoint = 'https://example.primehub.io/api/graphql' 15 | token = 'my-token' 16 | group = 'my-phusers' 17 | 18 | raw_config = self.create_config_dict(endpoint, token, group) 19 | default_cfg_path = self.create_fake_config(raw_config) 20 | 21 | class MyPrimeHubConfig(PrimeHubConfig): 22 | def get_default_path(self): 23 | return default_cfg_path 24 | 25 | # Verify config from the default path 26 | cfg = MyPrimeHubConfig() 27 | self.assertEqual(endpoint, cfg.endpoint) 28 | self.assertEqual(token, cfg.api_token) 29 | self.assertEqual(group, cfg.group) 30 | 31 | # Verify config from alternative path 32 | raw_config['endpoint'] = 'https://different-path.primehub.io' 33 | alternative_config_file = self.create_fake_config(raw_config) 34 | cfg = MyPrimeHubConfig(config=alternative_config_file) 35 | self.assertEqual('https://different-path.primehub.io', cfg.endpoint) 36 | self.assertEqual(cfg.get_default_path(), default_cfg_path) 37 | self.assertEqual(cfg.config_file, alternative_config_file) 38 | 39 | # Verify config override by ENV 40 | os.environ['PRIMEHUB_GROUP'] = 'env:group' 41 | os.environ['PRIMEHUB_API_TOKEN'] = 'env:api-token' 42 | os.environ['PRIMEHUB_API_ENDPOINT'] = 'env:endpoint' 43 | 44 | cfg.load_config_from_env() # force reload env-config 45 | self.assert_config_with_prefix('env', cfg) 46 | 47 | # Verify override from user 48 | cfg.endpoint = 'user:endpoint' 49 | cfg.api_token = 'user:api-token' 50 | cfg.group = 'user:group' 51 | self.assert_config_with_prefix('user', cfg) 52 | 53 | def test_primehub_config_save(self): 54 | endpoint = 'file:endpoint' 55 | token = 'file:api-token' 56 | group = 'file:group' 57 | 58 | # Arrange the original configuration file 59 | raw_config = self.create_config_dict(endpoint, token, group) 60 | config_file = self.create_fake_config(raw_config) 61 | cfg = PrimeHubConfig(config=config_file) 62 | self.assert_config_with_prefix('file', cfg) 63 | 64 | # Assume the user updating the configuration and persist it to the file 65 | cfg.endpoint = 'save:endpoint' 66 | cfg.api_token = 'save:api-token' 67 | cfg.group = 'save:group' 68 | 69 | fd, new_config_path = mkstemp('.json', text=True) 70 | cfg.save(new_config_path) 71 | 72 | # Verify the new configuration becoming with `save:*` values 73 | self.assert_config_with_prefix('save', cfg) 74 | 75 | def test_set_from_constructor(self): 76 | endpoint = 'file:endpoint' 77 | token = 'file:api-token' 78 | group = 'file:group' 79 | 80 | # Arrange the original configuration file 81 | raw_config = self.create_config_dict(endpoint, token, group) 82 | config_file = self.create_fake_config(raw_config) 83 | cfg = PrimeHubConfig(config=config_file, endpoint='prop:endpoint') 84 | self.assertEqual('prop:endpoint', cfg.endpoint) 85 | self.assertEqual('file:api-token', cfg.api_token) 86 | self.assertEqual('file:group', cfg.group) 87 | -------------------------------------------------------------------------------- /tests/test_datasets.py: -------------------------------------------------------------------------------- 1 | from primehub.datasets import validate_creation, validate_update, get_phfs_path, protect_metadata 2 | from primehub.utils import PrimeHubException, DatasetsException 3 | from tests import BaseTestCase 4 | 5 | 6 | class TestDatasets(BaseTestCase): 7 | 8 | def setUp(self) -> None: 9 | super(TestDatasets, self).setUp() 10 | 11 | def check_exception(self, cfg: dict, message: str, validator): 12 | with self.assertRaises(PrimeHubException) as context: 13 | validator(cfg) 14 | 15 | self.assertTrue(isinstance(context.exception, PrimeHubException)) 16 | self.assertEqual(message, context.exception.args[0]) 17 | 18 | def test_creation_validator(self): 19 | invalid_id_format_msg = "id should be string type and lower case alphanumeric characters, '-' or '.'. " \ 20 | "The value must start and end with an alphanumeric character " \ 21 | "and its length of it should be less than 63." 22 | 23 | self.check_exception({}, 'id is a required field', validate_creation) 24 | self.check_exception({'id': 123}, invalid_id_format_msg, validate_creation) 25 | self.check_exception({'id': '^.<', 'groupName': 'my-group'}, invalid_id_format_msg, validate_creation) 26 | self.check_exception({'id': '1_2_3', 'groupName': 'my-group'}, invalid_id_format_msg, validate_creation) 27 | 28 | self.check_exception({'id': 'dataset-1', 'tags': [123], 'groupName': 'my-group'}, 29 | 'The value of the tags should be a list of string.', validate_creation) 30 | self.check_exception({'id': 'dataset-1', 'tags': ['tag-1', 123], 'groupName': 'my-group'}, 31 | 'The value of the tags should be a list of string.', validate_creation) 32 | 33 | # pass with valid configuration 34 | validate_creation({'id': 'dataset-1', 'groupName': 'my-group'}) 35 | validate_creation({'id': 'dataset-1', 'groupName': 'my-group', 'tags': ['tag-1', 'tag-2']}) 36 | 37 | def test_update_validator(self): 38 | self.check_exception({'tags': [123], 'groupName': 'my-group'}, 39 | 'The value of the tags should be a list of string.', validate_update) 40 | self.check_exception({'tags': ['tag-1', 123], 'groupName': 'my-group'}, 41 | 'The value of the tags should be a list of string.', validate_update) 42 | 43 | # pass with valid configuration 44 | validate_update({}) 45 | validate_update({'tags': ['tag-1', 'tag-2']}) 46 | 47 | def test_gen_datasets_path(self): 48 | self.assertEqual(get_phfs_path('dataset-1', '/'), '/datasets/dataset-1/') 49 | self.assertEqual(get_phfs_path('dataset-1', './'), '/datasets/dataset-1/') 50 | self.assertEqual(get_phfs_path('dataset-1', '.'), '/datasets/dataset-1/') 51 | self.assertEqual(get_phfs_path('dataset-1', '.abc'), '/datasets/dataset-1/.abc') 52 | self.assertEqual(get_phfs_path('dataset-1', './a/../.abc'), '/datasets/dataset-1/a/../.abc') 53 | self.assertEqual(get_phfs_path('dataset-1', '///a/../.abc'), '/datasets/dataset-1///a/../.abc') 54 | 55 | def test_protect_metadata(self): 56 | dataset = 'dataset-1' 57 | # path to metadata 58 | with self.assertRaises(DatasetsException) as e: 59 | protect_metadata(dataset, get_phfs_path(dataset, '/.dataset')) 60 | self.assertEqual('Invalid Operation', str(e.exception)) 61 | 62 | with self.assertRaises(DatasetsException) as e: 63 | protect_metadata(dataset, get_phfs_path(dataset, '//.dataset')) 64 | self.assertEqual('Invalid Operation', str(e.exception)) 65 | 66 | with self.assertRaises(DatasetsException) as e: 67 | protect_metadata(dataset, get_phfs_path(dataset, '.dataset')) 68 | self.assertEqual('Invalid Operation', str(e.exception)) 69 | 70 | with self.assertRaises(DatasetsException) as e: 71 | protect_metadata(dataset, get_phfs_path(dataset, './a/../.dataset')) 72 | self.assertEqual('Invalid Operation', str(e.exception)) 73 | 74 | with self.assertRaises(DatasetsException) as e: 75 | protect_metadata(dataset, get_phfs_path(dataset, './.dataset')) 76 | self.assertEqual('Invalid Operation', str(e.exception)) 77 | 78 | with self.assertRaises(DatasetsException) as e: 79 | protect_metadata(dataset, get_phfs_path(dataset, '///a/../.dataset')) 80 | self.assertEqual('Invalid Operation', str(e.exception)) 81 | 82 | with self.assertRaises(DatasetsException) as e: 83 | protect_metadata(dataset, get_phfs_path(dataset, '///a/..//.dataset')) 84 | self.assertEqual('Invalid Operation', str(e.exception)) 85 | 86 | # pass with valid path 87 | protect_metadata(dataset, get_phfs_path(dataset, '/test.csv')) 88 | protect_metadata(dataset, get_phfs_path(dataset, '/dataset.csv')) 89 | protect_metadata(dataset, get_phfs_path(dataset, '/.data')) 90 | protect_metadata(dataset, get_phfs_path(dataset, '/dataset')) 91 | protect_metadata(dataset, get_phfs_path(dataset, '/.dataset.dataset')) 92 | protect_metadata(dataset, get_phfs_path(dataset, '/deep/.dataset')) 93 | -------------------------------------------------------------------------------- /tests/test_graphql_lint.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import os 3 | import re 4 | import textwrap 5 | from unittest import TestCase 6 | 7 | from tests.graphql_formatter import is_formatter_available, format_graphql 8 | 9 | 10 | def is_run_in_ci(): 11 | return os.environ.get('CI', 'false') == 'true' 12 | 13 | 14 | class GraphQLQueryFormatChecker(ast.NodeTransformer): 15 | 16 | def __init__(self, testutil: TestCase, filename: str): 17 | self.testutil = testutil 18 | self.filename = filename 19 | 20 | def visit_Assign(self, node): 21 | 22 | try: 23 | n = node.targets[0] 24 | if not hasattr(n, 'id'): 25 | return node 26 | 27 | if n.id != 'query': 28 | return node 29 | 30 | first_line_indent = self.get_indent(node) 31 | 32 | # show progress in CI mode 33 | if is_run_in_ci(): 34 | print(f'check {self.filename}:{n.lineno}') 35 | 36 | gql = strip_blank_line(node.value.s) 37 | if has_checkpoint(f'{self.filename}:{n.lineno}', gql): 38 | # skip by checkpoint 39 | if is_run_in_ci(): 40 | print('skip by checkpoint') 41 | 42 | return node 43 | 44 | formatted = textwrap.indent(format_graphql(gql), ' ' * first_line_indent) 45 | formatted = strip_blank_line(formatted) 46 | if gql != formatted: 47 | print(f'check {self.filename}:{n.lineno}') 48 | print(f'please replace the content by this:\n\n>>>>\n{formatted}\n<<<<\n') 49 | else: 50 | save_checkpoint(f'{self.filename}:{n.lineno}', gql) 51 | self.testutil.assertEqual(gql, formatted) 52 | except BaseException as e: 53 | if isinstance(e, AssertionError): 54 | raise e 55 | print(type(e), e) 56 | 57 | return node 58 | 59 | def get_indent(self, node): 60 | first_line_indent = 0 61 | for line in node.value.s.split('\n'): 62 | if line.strip() == '': 63 | continue 64 | m = re.search(r'^(\s+).*', line) 65 | if m: 66 | first_line_indent = len(m.group(1)) 67 | break 68 | return first_line_indent 69 | 70 | 71 | def strip_blank_line(formatted): 72 | return '\n'.join([x for x in formatted.split('\n') if x.strip() != '']) 73 | 74 | 75 | def cache_location(content, file_loc): 76 | os.makedirs('/tmp/graphql_lint', 0o777, exist_ok=True) 77 | filepath = os.path.join('/tmp/graphql_lint', f'._gql_check_{checksum(file_loc)}.{checksum(content)}') 78 | return filepath 79 | 80 | 81 | def save_checkpoint(file_loc, content): 82 | filepath = cache_location(content, file_loc) 83 | with open(filepath, 'w') as fh: 84 | fh.write('') 85 | 86 | 87 | def has_checkpoint(file_loc, content): 88 | filepath = cache_location(content, file_loc) 89 | return os.path.exists(filepath) 90 | 91 | 92 | def checksum(content: str): 93 | import hashlib 94 | m = hashlib.md5(content.encode()) 95 | return m.hexdigest() 96 | 97 | 98 | def source_contents(): 99 | project_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '../primehub')) 100 | for root, dirs, files in os.walk(project_dir): 101 | for f in files: 102 | if f.endswith('.py'): 103 | p = os.path.join(root, f) 104 | with open(p, 'r') as fh: 105 | content = fh.read() 106 | yield p, content 107 | break 108 | 109 | 110 | class GraphQLLint(TestCase): 111 | 112 | def test_graphql_lint(self): 113 | if not is_formatter_available(): 114 | print("GRAPHQL FORMATTER NOT AVAILABLE. (please install prettier first)") 115 | print("SKIP GRAPHQL LINT") 116 | return 117 | 118 | print("\n---- start to lint graphql ----") 119 | for filename, c in source_contents(): 120 | node = ast.parse(c) 121 | GraphQLQueryFormatChecker(self, filename).visit(node) 122 | -------------------------------------------------------------------------------- /tests/test_group_resource_operation.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from primehub import Module, cmd, Helpful 4 | from primehub.resource_operations import GroupResourceOperation 5 | from tests import BaseTestCase 6 | 7 | 8 | class GroupFoobarCommand(Helpful, Module, GroupResourceOperation): 9 | 10 | def help_description(self): 11 | return "foo bar" 12 | 13 | @cmd(name='list', description='') 14 | def list(self): 15 | return self.do_list('fake-query', 'foobar') 16 | 17 | @cmd(name='get', description='') 18 | def get(self, name): 19 | return self.do_get('fake-query', 'foobar', name) 20 | 21 | 22 | class TestGroupResourceOperation(BaseTestCase): 23 | 24 | def setUp(self) -> None: 25 | super(TestGroupResourceOperation, self).setUp() 26 | self.sdk.register_command('test_group_resource_operation', GroupFoobarCommand) 27 | 28 | def test_group_not_found(self): 29 | output = self.cli_stderr(['app.py', 'test_group_resource_operation', 'list']) 30 | self.assertEqual('No group information, please configure the active group first.', output.strip()) 31 | 32 | def test_group_resource_operation(self): 33 | self.sdk.primehub_config.group_info = {'name': 'phusers', 'id': 'any-id'} 34 | self.mock_request.return_value = { 35 | 'data': { 36 | 'me': { 37 | 'effectiveGroups': [ 38 | {'name': 'group-1', 'foobar': [dict(name="1", value="1")]}, 39 | {'name': 'group-2', 'foobar': [dict(name="2", value="2")]}, 40 | {'name': 'phusers', 'foobar': [dict(name="3", value="3")]}, 41 | ] 42 | } 43 | } 44 | } 45 | 46 | # verify list will select the right group 47 | output = self.cli_stdout(['app.py', 'test_group_resource_operation', 'list']) 48 | self.assertEqual([dict(name="3", value="3")], json.loads(output)) 49 | 50 | # verify get by name 51 | output = self.cli_stdout(['app.py', 'test_group_resource_operation', 'get', '3']) 52 | self.assertEqual(dict(name="3", value="3"), json.loads(output)) 53 | -------------------------------------------------------------------------------- /tests/test_http_utils.py: -------------------------------------------------------------------------------- 1 | from types import GeneratorType 2 | 3 | from primehub import Client 4 | from tests import BaseTestCase 5 | 6 | 7 | class TestHttpRequestLogs(BaseTestCase): 8 | 9 | def setUp(self) -> None: 10 | super(TestHttpRequestLogs, self).setUp() 11 | 12 | def test_request_logs(self): 13 | url = 'https://github.com/InfuseAI/primehub-python-sdk' 14 | c = Client(self.sdk.primehub_config) 15 | 16 | # with follow 17 | g = c.request_logs(url, True, 0) 18 | self.assertIsInstance(g, GeneratorType) 19 | count = 0 20 | for x in g: 21 | count = count + 1 22 | self.assertTrue(count > 1, 'Generator gives lots of lines') 23 | 24 | # no follow 25 | g = c.request_logs(url, False, 0) 26 | self.assertIsInstance(g, GeneratorType) 27 | count = 0 28 | for x in g: 29 | count = count + 1 30 | self.assertTrue(count > 1, 'Generator gives lots of lines') 31 | -------------------------------------------------------------------------------- /tests/test_implementation_in_source_level.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import os 3 | 4 | from primehub.utils.decorators import __requires_permission__ 5 | from tests import BaseTestCase 6 | 7 | 8 | class TestAskForPermissionMethods(BaseTestCase): 9 | 10 | def test_verify_ask_for_permissions(self): 11 | commands = __requires_permission__.keys() 12 | for cmd in commands: 13 | 14 | if "test" in cmd: 15 | continue 16 | # cmd will look like 'primehub.schedules.delete' 17 | package_name, command_name, method_name = cmd.split(".") 18 | 19 | if ":" in command_name: 20 | continue 21 | 22 | if package_name != 'primehub': 23 | # only cares about primehub package 24 | continue 25 | 26 | cmd_obj = None 27 | if command_name in self.sdk.commands: 28 | cmd_obj = self.sdk.commands[command_name] 29 | elif command_name in self.sdk.admin_commands: 30 | cmd_obj = self.sdk.admin_commands[command_name] 31 | 32 | if cmd_obj is None: 33 | continue 34 | 35 | method = getattr(cmd_obj, method_name) 36 | 37 | # check the "**kwargs" in the last parameter 38 | signature = inspect.signature(method) 39 | signature = [str(x) for x in signature.parameters.values()] 40 | 41 | if not signature: 42 | self.report_bad_method_signature(cmd, method) 43 | 44 | if signature[-1] != '**kwargs': 45 | self.report_bad_method_signature(cmd, method) 46 | 47 | def report_bad_method_signature(self, cmd, method): 48 | src, lines = inspect.getsourcelines(method) 49 | delimeter = '.' * 80 50 | report = f'\n{delimeter} \nCommand: {cmd} \nLine: {lines}\n{"".join(src[:5])}\n{delimeter}' 51 | raise ValueError( 52 | f'@ask_for_permission must have a "**kwargs" in the last parameter, please update: {cmd}{report}') 53 | 54 | 55 | class TestDocumentation(BaseTestCase): 56 | """ 57 | There should be 3 document files for a command. 58 | 59 | A notebook: 60 | ./docs/notebook/{command}.ipynb 61 | 62 | A generated command line doc: 63 | ./docs/CLI/{command}.md 64 | 65 | An example will embedded into cli doc: 66 | ./primehub/extras/templates/examples/{command}.ipynb 67 | """ 68 | 69 | def test_docs(self): 70 | project_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) 71 | 72 | def generate_doc_paths(command): 73 | files = [ 74 | f'./docs/notebook/{command}.ipynb', 75 | f'./docs/CLI/{command}.md', 76 | f'./primehub/extras/templates/examples/{command}.md' 77 | ] 78 | return [os.path.normpath(os.path.join(project_dir, x)) for x in files] 79 | 80 | for command in self.sdk.commands: 81 | # skip hidden command (starts with :) 82 | if command and command.startswith(':'): 83 | continue 84 | 85 | # skip version command 86 | if command == 'version': 87 | continue 88 | if command == 'admin': 89 | continue 90 | 91 | notebook, cli, cli_example = generate_doc_paths(command) 92 | 93 | if not os.path.exists(notebook): 94 | raise ValueError(f'There should have a {notebook} for command "{command}"') 95 | 96 | if not os.path.exists(cli): 97 | raise ValueError(f'There should have a {cli} for command "{command}"') 98 | 99 | if not os.path.exists(cli_example): 100 | raise ValueError(f'There should have a {cli_example} for command "{command}"') 101 | -------------------------------------------------------------------------------- /tests/test_jobs.py: -------------------------------------------------------------------------------- 1 | from primehub.jobs import verify_basic_field, verify_timeout 2 | from primehub.jobs import invalid_config 3 | from primehub.utils import PrimeHubException 4 | from tests import BaseTestCase 5 | 6 | 7 | class TestJobs(BaseTestCase): 8 | 9 | def setUp(self) -> None: 10 | super(TestJobs, self).setUp() 11 | 12 | def check_exception(self, cfg: dict, message: str, callback=None): 13 | with self.assertRaises(PrimeHubException) as context: 14 | if callback is None: 15 | verify_basic_field(cfg) 16 | else: 17 | callback(cfg) 18 | 19 | self.assertTrue(isinstance(context.exception, PrimeHubException)) 20 | self.assertEqual(message, context.exception.args[0]) 21 | 22 | def get_exception_message(self, message: str, callback): 23 | with self.assertRaises(PrimeHubException) as context: 24 | callback(message) 25 | 26 | return context.exception.args[0] 27 | 28 | def test_basic_field_validator(self): 29 | # check required fields 30 | exception_msg = self.get_exception_message('displayName is required', invalid_config) 31 | self.check_exception({}, exception_msg) 32 | 33 | exception_msg = self.get_exception_message('command is required', invalid_config) 34 | self.check_exception({'instanceType': 'cpu-1', 'image': 'base-notebook', 'displayName': 'job-example'}, 35 | exception_msg) 36 | 37 | # check invalid fields 38 | self.check_exception({'instanceType': 123, 'image': 'base-notebook', 'displayName': 'job-example', 39 | 'command': 'echo \"great job\"'}, 'instanceType should be string value') 40 | 41 | self.check_exception({'instanceType': 'cpu-1', 'image': '', 'displayName': 'job-example', 42 | 'command': 'echo \"great job\"'}, 'image should be specified') 43 | 44 | self.check_exception({'instanceType': 'cpu-1', 'image': None, 'displayName': 'job-example', 45 | 'command': 'echo \"great job\"'}, 'image should be string value') 46 | 47 | # pass with valid config 48 | verify_basic_field({'instanceType': 'cpu-1', 'image': 'base-notebook', 'displayName': 'job-example', 49 | 'command': 'echo \"great job\"'}) 50 | 51 | def test_timeout_validator(self): 52 | # check 53 | self.check_exception({'activeDeadlineSeconds': '86400'}, 'activeDeadlineSeconds should be int value', 54 | verify_timeout) 55 | self.check_exception({'activeDeadlineSeconds': None}, 'activeDeadlineSeconds should not be empty', 56 | verify_timeout) 57 | 58 | verify_timeout({'activeDeadlineSeconds': 86400}) 59 | -------------------------------------------------------------------------------- /tests/test_output_utils.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from primehub import Module 4 | 5 | 6 | class OutputUtilsTest(TestCase): 7 | 8 | def test_output(self): 9 | self.assertEqual(['value'], Module.output({'data': {'me': ['value']}}, 'me')) 10 | self.assertEqual('result', Module.output({'data': {'1': {'2': 'result'}}}, '1.2')) 11 | 12 | # if there is no data, return all payload 13 | self.assertEqual({'error': ['messages']}, Module.output({'error': ['messages']}, 'any.path.to.data')) 14 | -------------------------------------------------------------------------------- /tests/test_print.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import os 3 | from unittest import TestCase 4 | 5 | 6 | class PrintChecker(ast.NodeTransformer): 7 | 8 | def __init__(self, testutil: TestCase, filename: str): 9 | self.testutil = testutil 10 | self.filename = filename 11 | 12 | def visit_Call(self, node): 13 | if isinstance(node.func, ast.Name) and node.func.id == 'print': 14 | file_kw_arg = [x for x in node.keywords if x.arg == 'file'] 15 | if len(file_kw_arg) == 0: 16 | message = f'print should have "file" keyword arg: {self.filename}:{node.lineno}:{node.col_offset}\n' 17 | message += 'fix it with file=primehub.stderr (it could be primehub.stdout)' 18 | self.testutil.fail(message) 19 | return node 20 | 21 | 22 | def source_contents(): 23 | project_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '../primehub')) 24 | for root, dirs, files in os.walk(project_dir): 25 | for f in files: 26 | if f.endswith('.py'): 27 | p = os.path.join(root, f) 28 | with open(p, 'r') as fh: 29 | content = fh.read() 30 | yield p, content 31 | break 32 | 33 | 34 | class PrintTest(TestCase): 35 | 36 | def test_print_with_file_arg(self): 37 | for filename, c in source_contents(): 38 | node = ast.parse(c) 39 | PrintChecker(self, filename).visit(node) 40 | -------------------------------------------------------------------------------- /tests/test_recurring_jobs.py: -------------------------------------------------------------------------------- 1 | from primehub.recurring_jobs import invalid_config 2 | from primehub.recurring_jobs import verify_config, verify_recurrence, verify_recurrence_options 3 | from primehub.utils import PrimeHubException 4 | from tests import BaseTestCase 5 | 6 | 7 | class TestRecurringJobs(BaseTestCase): 8 | 9 | def setUp(self) -> None: 10 | super(TestRecurringJobs, self).setUp() 11 | 12 | def check_exception(self, cfg: dict, message: str, callback=None): 13 | with self.assertRaises(PrimeHubException) as context: 14 | if callback is None: 15 | verify_recurrence(cfg) 16 | else: 17 | callback(cfg) 18 | 19 | self.assertTrue(isinstance(context.exception, PrimeHubException)) 20 | self.assertEqual(message, context.exception.args[0]) 21 | 22 | def get_exception_message(self, message: str, callback): 23 | with self.assertRaises(PrimeHubException) as context: 24 | callback(message) 25 | 26 | return context.exception.args[0] 27 | 28 | def test_config_update_validator(self): 29 | verify_config({'instanceType': 'cpu-1'}, True) 30 | verify_config({}, True) 31 | 32 | def test_recurrence_validator(self): 33 | # check required fields 34 | exception_msg = self.get_exception_message('recurrence is required', invalid_config) 35 | self.check_exception({}, exception_msg) 36 | 37 | self.check_exception({'recurrence': 'type: daily, cron: 0 4 * * *'}, 'recurrence should be a json object') 38 | 39 | verify_recurrence({'recurrence': {'type': 'custom', 'cron': '0 4 * * *'}}) 40 | 41 | def test_recurrence_options_validator(self): 42 | self.check_exception({'type': 123}, 'type should be string value', verify_recurrence_options) 43 | self.check_exception({'type': 'my custom'}, '\'my custom\' is not acceptable type', verify_recurrence_options) 44 | 45 | exception_msg = self.get_exception_message('type is required', invalid_config) 46 | self.check_exception({'cron': '0 4 * * *'}, exception_msg, verify_recurrence_options) 47 | exception_msg = self.get_exception_message('type is required', invalid_config) 48 | self.check_exception({'type: '', cron': '0 4 * * *'}, exception_msg, verify_recurrence_options) 49 | 50 | self.check_exception({'type': 'custom', 'cron': 5566}, 'cron should be string value', verify_recurrence_options) 51 | exception_msg = self.get_exception_message('cron is required in custom type', invalid_config) 52 | self.check_exception({'type': 'custom'}, exception_msg, verify_recurrence_options) 53 | 54 | verify_recurrence_options({'type': 'on-demand', 'cron': ''}) 55 | verify_recurrence_options({'type': 'weekly', 'cron': ''}) 56 | verify_recurrence_options({'type': 'monthly'}) 57 | -------------------------------------------------------------------------------- /tests/test_sdk_to_admin_cli.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from primehub import Helpful, Module, cmd 4 | from tests import BaseTestCase 5 | 6 | 7 | class FakeCommand(Helpful, Module): 8 | 9 | @cmd(name='list', description='action_no_args') 10 | def action_no_args(self): 11 | return [1, 2, 3] 12 | 13 | def help_description(self): 14 | return "helpful" 15 | 16 | 17 | class TestAdminCommandGroupToCommandLine(BaseTestCase): 18 | 19 | def setUp(self) -> None: 20 | super(TestAdminCommandGroupToCommandLine, self).setUp() 21 | 22 | # clean commands, add the FakeCommand 23 | self.sdk.register_admin_command('test_datasets', FakeCommand) 24 | 25 | def fake(self) -> FakeCommand: 26 | return FakeCommand(self.sdk) 27 | 28 | def test_primehub_admin(self): 29 | output = self.cli_stdout(['app.py', 'admin', 'test_datasets', 'list']) 30 | self.assertEqual([1, 2, 3], json.loads(output)) 31 | 32 | def test_primehub_admin_help(self): 33 | output = self.cli_stderr(['app.py', 'admin', 'test_datasets', '-h']) 34 | usage = [x.strip() for x in output.split('\n') if '' in x][0] 35 | self.assertEqual('primehub admin test_datasets ', usage) 36 | -------------------------------------------------------------------------------- /tests/test_sdk_to_cli_module_alias.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from primehub import Helpful, Module, cmd 4 | from tests import BaseTestCase 5 | 6 | 7 | class FakeCommandBehindAlias(Helpful, Module): 8 | 9 | @cmd(name='cmd-no-args', description='action_no_args') 10 | def action_no_args(self): 11 | return dict(name='action_no_args') 12 | 13 | def help_description(self): 14 | return "help message for fake-command" 15 | 16 | 17 | class TestCommandGroupAlias(BaseTestCase): 18 | 19 | def setUp(self) -> None: 20 | super(TestCommandGroupAlias, self).setUp() 21 | 22 | # clean commands, add the FakeCommand 23 | self.sdk.register_command('module_name_with_dash_alias', FakeCommandBehindAlias, 'module-name-with-dash-alias') 24 | 25 | def fake(self) -> FakeCommandBehindAlias: 26 | return FakeCommandBehindAlias(self.sdk) 27 | 28 | def test_command_groups_with_cli(self): 29 | output = self.cli_stdout(['app.py', 'module-name-with-dash-alias', 'cmd-no-args']) 30 | self.assertEqual(dict(name='action_no_args'), json.loads(output)) 31 | 32 | def test_command_groups_with_sdk(self): 33 | output = self.sdk.module_name_with_dash_alias.action_no_args() 34 | self.assertEqual(dict(name='action_no_args'), output) 35 | -------------------------------------------------------------------------------- /tests/test_test_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from tests import BaseTestCase 4 | 5 | 6 | class TestHttpMocked(BaseTestCase): 7 | 8 | def setUp(self) -> None: 9 | super(TestHttpMocked, self).setUp() 10 | 11 | def test_mocked_me(self): 12 | self.mock_request.return_value = {'data': {'me': {'foo': 'barbar'}}} 13 | self.assertEqual(dict(foo='barbar'), json.loads(self.cli_stdout(['app.py', 'me']))) 14 | 15 | 16 | class TestStdoutStderrSideEffect(BaseTestCase): 17 | 18 | def setUp(self) -> None: 19 | super(TestStdoutStderrSideEffect, self).setUp() 20 | 21 | def test_stdout_stderr_side_effect_removed(self): 22 | self.mock_request.return_value = {'data': {'me': {'message': 'side-effect'}}} 23 | 24 | output = self.cli_stdout(['app.py', 'me']) 25 | self.assertEqual({'message': 'side-effect'}, json.loads(output)) 26 | 27 | output = self.cli_stdout(['app.py', 'me']) 28 | self.assertEqual({'message': 'side-effect'}, json.loads(output)) 29 | 30 | help1 = self.cli_stderr(['app.py']) 31 | help2 = self.cli_stderr(['app.py']) 32 | self.assertEqual(help1, help2) 33 | -------------------------------------------------------------------------------- /tests/test_volumes.py: -------------------------------------------------------------------------------- 1 | from primehub.admin_volumes import validate, validate_creation 2 | from primehub.utils import PrimeHubException 3 | from tests import BaseTestCase 4 | 5 | 6 | class TestVolumes(BaseTestCase): 7 | 8 | def setUp(self) -> None: 9 | super(TestVolumes, self).setUp() 10 | 11 | def check_required(self, input: dict, message: str): 12 | with self.assertRaises(PrimeHubException) as context: 13 | validate(input) 14 | 15 | self.assertTrue(isinstance(context.exception, PrimeHubException)) 16 | self.assertEqual(message, context.exception.args[0]) 17 | 18 | def test_validator(self): 19 | # check required fields 20 | self.check_required({}, 'name is required') 21 | self.check_required({'name': 'volume-name'}, 'type is required') 22 | 23 | # check formats 24 | self.check_required({'name': '-name', 'type': 'pv'}, 25 | "[name] should be lower case alphanumeric characters, '-' or '.', " 26 | "and must start and end with an alphanumeric character.") 27 | 28 | self.check_required({'name': 'name', 'type': 'whatever'}, 29 | "[type] should be one of ['pv', 'nfs', 'hostPath', 'git', 'env']") 30 | 31 | # check writable groups 32 | self.check_required({'name': 'name', 'type': 'git', 'enableUploadServer': False}, 33 | "[enableUploadServer] only can use with should be one of ['pv', 'nfs', 'hostPath'] types") 34 | 35 | # check groups connect/disconnect 36 | self.check_required({'name': 'name', 'type': 'pv', 'groups': {'connect': [{'name': 'my-group'}]}}, 37 | "group connect should be a pair {id, writable}") 38 | 39 | self.check_required( 40 | {'name': 'name', 'type': 'pv', 'groups': {'disconnect': [{'id': 'my-id', 'writable': True}]}}, 41 | "disconnect connect should be an entry {id}") 42 | 43 | def check_creation_required(self, input: dict, message: str): 44 | with self.assertRaises(PrimeHubException) as context: 45 | validate_creation(input) 46 | 47 | self.assertTrue(isinstance(context.exception, PrimeHubException)) 48 | self.assertEqual(message, context.exception.args[0]) 49 | 50 | def test_pv_create_validator(self): 51 | # check required fields 52 | self.check_creation_required({'name': 'name', 'type': 'pv'}, 53 | "pvProvisioning is required for pv type " 54 | "and its value should be one of ['auto', 'manual']") 55 | 56 | self.check_creation_required({'name': 'name', 'type': 'pv', 'pvProvisioning': 'no-such-way'}, 57 | "pvProvisioning is required for pv type " 58 | "and its value should be one of ['auto', 'manual']") 59 | 60 | valid_input = {'name': 'name', 'type': 'pv', 'pvProvisioning': 'auto'} 61 | self.assertEqual(valid_input, validate_creation(valid_input)) 62 | 63 | def test_nfs_create_validator(self): 64 | # check required fields 65 | self.check_creation_required({'name': 'name', 'type': 'nfs'}, 66 | "nfsServer and nfsPath are required for nfs type") 67 | 68 | self.check_creation_required({'name': 'name', 'type': 'nfs', 'nfsServer': '127.0.0.1'}, 69 | "nfsServer and nfsPath are required for nfs type") 70 | 71 | valid_input = {'name': 'name', 'type': 'nfs', 'nfsServer': '127.0.0.1', 'nfsPath': '/data'} 72 | self.assertEqual(valid_input, validate_creation(valid_input)) 73 | 74 | def test_hostPath_create_validator(self): 75 | # check required fields 76 | self.check_creation_required({'name': 'name', 'type': 'hostPath'}, 77 | "hostPath is required for hostPath type") 78 | 79 | valid_input = {'name': 'name', 'type': 'hostPath', 'hostPath': '/data'} 80 | self.assertEqual(valid_input, validate_creation(valid_input)) 81 | 82 | def test_git_create_validator(self): 83 | # check required fields 84 | self.check_creation_required({'name': 'name', 'type': 'git'}, 85 | "url is required for git type") 86 | 87 | valid_input = {'name': 'name', 'type': 'git', 'url': 'https://github.com/InfuseAI/primehub-python-sdk'} 88 | self.assertEqual(valid_input, validate_creation(valid_input)) 89 | --------------------------------------------------------------------------------