├── .gitignore ├── Development.rst ├── Jenkinsfile ├── LICENSE ├── README.rst ├── docker ├── Dockerfile ├── Makefile ├── README.md └── run_tests.sh ├── docs ├── Makefile ├── conf.py ├── hops.rst ├── imgs │ ├── add-python-module.png │ ├── driver.png │ ├── executor-hdfs-log.png │ ├── executor-stderr1.png │ ├── executor-stderr2.png │ ├── executor-stderr3.png │ ├── executor-stderr4.png │ └── hopsml-pyspark.png ├── index.rst ├── license.rst ├── modules.rst ├── readme.rst └── source │ ├── hops.rst │ └── modules.rst ├── hops ├── __init__.py ├── constants.py ├── credentials_provider.py ├── dataset.py ├── devices.py ├── elasticsearch.py ├── exceptions.py ├── experiment.py ├── experiment_impl │ ├── __init__.py │ ├── distribute │ │ ├── __init__.py │ │ ├── mirrored.py │ │ ├── mirrored_reservation.py │ │ ├── mirrored_reservation_client.py │ │ ├── parameter_server.py │ │ ├── parameter_server_client.py │ │ └── parameter_server_reservation.py │ ├── launcher.py │ ├── parallel │ │ ├── __init__.py │ │ ├── differential_evolution.py │ │ ├── grid_search.py │ │ └── random_search.py │ └── util │ │ ├── __init__.py │ │ └── experiment_utils.py ├── featurestore.py ├── hdfs.py ├── hive.py ├── jobs.py ├── kafka.py ├── model.py ├── numpy_helper.py ├── pandas_helper.py ├── project.py ├── secret.py ├── service_discovery.py ├── serving.py ├── tensorboard.py ├── tests │ └── test_resources │ │ ├── attendances_features.csv │ │ ├── featuregroup.json │ │ ├── featurestore_metadata.json │ │ ├── games_features.csv │ │ ├── mnist │ │ └── img_1.jpg │ │ ├── online_featurestore_connector.json │ │ ├── players_features.csv │ │ ├── season_scores_features.csv │ │ ├── spark-avro_2.11-2.4.0.jar │ │ ├── spark-tensorflow-connector_2.11-1.12.0.jar │ │ ├── statistics.json │ │ ├── teams_features.csv │ │ ├── token.jwt │ │ └── training_dataset.json ├── tls.py ├── user.py ├── util.py ├── version.py └── xattr.py ├── imgs ├── add-python-module.png ├── driver.png ├── executor-hdfs-log.png ├── executor-stderr1.png ├── executor-stderr2.png ├── executor-stderr3.png ├── executor-stderr4.png ├── feature_plots1.png ├── feature_plots2.png └── hopsml-pyspark.png ├── it_tests └── integration_tests.ipynb ├── pytest.ini ├── setup.cfg ├── setup.py └── source ├── hops.featurestore_impl.dao.common.rst ├── hops.featurestore_impl.dao.datasets.rst ├── hops.featurestore_impl.dao.featuregroups.rst ├── hops.featurestore_impl.dao.features.rst ├── hops.featurestore_impl.dao.featurestore.rst ├── hops.featurestore_impl.dao.rst ├── hops.featurestore_impl.dao.settings.rst ├── hops.featurestore_impl.dao.stats.rst ├── hops.featurestore_impl.dao.storageconnectors.rst ├── hops.featurestore_impl.exceptions.rst ├── hops.featurestore_impl.featureframes.rst ├── hops.featurestore_impl.online_featurestore.rst ├── hops.featurestore_impl.query_planner.rst ├── hops.featurestore_impl.rest.rst ├── hops.featurestore_impl.rst ├── hops.featurestore_impl.util.rst ├── hops.featurestore_impl.visualizations.rst ├── hops.rst ├── modules.rst └── setup.rst /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | *~ 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # dotenv 85 | .env 86 | 87 | # virtualenv 88 | .venv 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | .spyproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | 105 | #intelliJ 106 | .idea 107 | workspace.xml 108 | 109 | .cache 110 | 111 | .pytest_cache 112 | 113 | spark-warehouse/ 114 | 115 | training_datasets/ 116 | 117 | metastore_db 118 | .vscode/* 119 | -------------------------------------------------------------------------------- /Development.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Development 3 | =========== 4 | 5 | Setting up development environment 6 | --------------------------------------------------- 7 | This section shows a way to configure a development environment that allows you to run tests and build documentation. 8 | 9 | The recommended way to run the unit tests is to use the Dockerized Linux workspace via the Makefile provided at :code:`docker/Makefile`. 10 | The commands below will build the Docker image, start a running container with your local hops-util-py source code mounted 11 | into it from the host at :code:`/hops`. A python 3.6 environment is available in the container at :code:`/hops_venv3.6/`. 12 | Moreover, the commands below will also open a BASH shell into it (you must have GNU Make and Docker installed beforehand (:code:`sudo apt-get install docker.io`)): 13 | 14 | .. code-block:: bash 15 | 16 | cd docker # go to where Makefile is located 17 | 18 | # Build docker container and mount (you might need sudo), 19 | # this will fail if you already have a container named "hops" running 20 | # (if that is the case, just run `make shell` (or `sudo make shell`) to login to the existing container instead, 21 | # or if you want the kill the existing container and re-build, you can execute: `make clean` and then make build run shell) 22 | make build run shell # this might require sudo 23 | 24 | # Run the unit tests 25 | cd /hops/docker # once inside docker 26 | ./run_tests.sh 3.6 # for python 3.6 27 | ./run_tests.sh 3.6 -s # run tests with verbose flag 28 | 29 | # Alternatively you can skip the bash scripts and write the commands yourself (this gives you more control): 30 | cd /hops #inside the container 31 | source /hops_venv3.6/bin/activate # for python 3.6 32 | pip install -e . # install latest source 33 | pytest -v hops # run tests, note if you want to re-run just edit the code in your host and run the same command, you do not have to re-run pip install.. 34 | 35 | The docker approach is recommended if you already have an existing SPARK/Hadoop installation in your environment to avoid conflicts 36 | ( for example if you want to run the tests from inside a VM where Hops is deployed). Moreover, using docker makes it simpler to test python 3.6. 37 | 38 | Also note that when you edit files inside your local machine the changes will automatically get reflected in the docker 39 | container since the directory is mounted there so you can easily re-run tests during development as you do code-changes. 40 | 41 | To open up a shell in an already built :code:`hops` container: 42 | 43 | .. code-block:: bash 44 | 45 | cd docker # go to where Makefile is located 46 | make shell 47 | 48 | 49 | To kill an existing :code:`hops` container: 50 | 51 | 52 | .. code-block:: bash 53 | 54 | cd docker # go to where Makefile is located 55 | make clean 56 | 57 | If you want to run the tests in your local environment (watch out for spark/hadoop/environment conflicts) you can use the following setup: 58 | 59 | .. code-block:: bash 60 | 61 | sudo apt-get install python3-dev # install python3 62 | sudo apt-get install libsasl2-dev # for SSL used in kafka 63 | pip install virtualenv 64 | rm -rf env # remove old env if it exists 65 | virtualenv env --python #creates a virtual env for tests (e.g virtualenv env --python /usr/bin/python3.5) 66 | source env/bin/activate #activates the env 67 | pip install -U pip setuptools #install dependency 68 | pip install -e .[docs,tf,test] #install hops together with dependencies and extra dependencies for docs,tensorflow and tests 69 | pytest -v hops # run tests 70 | 71 | Unit tests 72 | ---------- 73 | To run unit tests locally: 74 | 75 | .. code-block:: bash 76 | 77 | pytest -v hops # Runs all tests 78 | pytest -v hops/tests/test_featurestore.py # Run specific test module 79 | pytest -v hops/tests/test_featurestore.py -k 'test_project_featurestore' # Run specific test in module 80 | pytest -m prepare # run test setups before parallel execution. **Note**: Feature store test suite is best run sequentially, otherwise race-conditions might cause errors. 81 | pytest -v hops -n 5 # Run tests in parallel with 5 workers. (Run prepare first) 82 | pytest -v hops -n auto #Run with automatically selected number of workers 83 | pytest -v hops -s # run with printouts (stdout) 84 | 85 | Documentation 86 | ------------- 87 | 88 | We use sphinx to automatically generate API-docs 89 | 90 | .. code-block:: bash 91 | 92 | pip install -e .[docs] 93 | cd docs; make html 94 | 95 | Integration Tests 96 | ------------- 97 | 98 | The notebooks in :code:`it_tests/` are used for integration testing by running them as jobs on a Hopsworks installation. 99 | The integration tests can be triggered from https://github.com/logicalclocks/hops-testing by using the following steps: 100 | 101 | 1. Open a PR in hops-testing and override the :code:`test_manifesto` with the cookbooks you want to test 102 | 2. In your PR, add the following attribute in your vagrantfiles to run the integration tests: :code:`test:hopsworks:it = true`, 103 | e.g in Vagrantfile-centos and Vagrantfile-ubuntu add: 104 | 105 | .. code-block:: bash 106 | 107 | config.vm.provision :chef_solo do |chef| 108 | chef.cookbooks_path = "cookbooks" 109 | chef.json = { 110 | "test" => { 111 | "hopsworks" => { 112 | "it" => true 113 | } 114 | } 115 | } 116 | 117 | 3. If you need to test a version of hops-util-py that is not merged you can set the chef attributes in your cluster-definition 118 | as follows to use a branch called :code:`test` in repository of :code:`kim/hops-util-py`: 119 | 120 | .. code-block:: bash 121 | 122 | conda: 123 | hops-util-py: 124 | install-mode: "git" 125 | branch: "test" 126 | repo: "kim" 127 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | node { 4 | label 'local' 5 | } 6 | } 7 | 8 | stages { 9 | stage ('setup') { 10 | steps { 11 | // Creates the virtualenv before proceeding 12 | sh """ 13 | if [ ! -d $WORKSPACE/../hops-util-py-env ]; 14 | then 15 | virtualenv --python=/usr/bin/python3.6 $WORKSPACE/../hops-util-py-env 16 | fi 17 | $WORKSPACE/../hops-util-py-env/bin/pip install twine sphinx sphinx-autobuild recommonmark sphinx_rtd_theme jupyter_sphinx_theme hops readme_renderer[md] boto3 18 | rm -rf dist/* 19 | """ 20 | } 21 | } 22 | stage ('build') { 23 | steps { 24 | sh """ 25 | source $WORKSPACE/../hops-util-py-env/bin/activate 26 | python ./setup.py sdist 27 | """ 28 | } 29 | } 30 | stage ('build-doc') { 31 | steps { 32 | sh """ 33 | source $WORKSPACE/../hops-util-py-env/bin/activate 34 | cd docs; sphinx-apidoc -f -o source/ ../hops ../hops/distribute/ ../hops/launcher.py ../hops/grid_search.py ../hops/differential_evolution.py ../hops/random_search.py ../hops/version.py ../hops/constants.py ../hops/featurestore_impl/; make html; cd .. 35 | """ 36 | } 37 | } 38 | //stage ('update-it-notebook') { 39 | // steps { 40 | // sh 'scp it_tests/integration_tests.ipynb snurran:/var/www/hops/hops-util-py_tests' 41 | // } 42 | // } 43 | stage ('deploy-bin') { 44 | environment { 45 | PYPI = credentials('977daeb0-e1c8-43a0-b35a-fc37bb9eee9b') 46 | } 47 | steps { 48 | sh """ 49 | source $WORKSPACE/../hops-util-py-env/bin/activate 50 | twine upload -u $PYPI_USR -p $PYPI_PSW --skip-existing dist/* 51 | """ 52 | } 53 | } 54 | stage ('deploy-doc') { 55 | steps { 56 | sh """ 57 | set -x 58 | version=`curl https://hops-py.logicalclocks.com/ --silent | grep -A1 "\\"version\\"" | tail -1` 59 | current_version=`cat hops/version.py | head -1 | awk -F '=' {'print \$2'}` 60 | current_version=`echo \${current_version//\\'}` 61 | 62 | if [ `printf '%s\n' "\$version" "\$current_version" | sort -V | tail -1` == \$current_version ]; 63 | then 64 | echo "Ignored" 65 | else 66 | rm -r /opt/hops-py/* 67 | cp -r docs/_build/html/* /opt/hops-py/ 68 | fi 69 | """ 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | # This container installs hops-util-py and necessary dependencies to run tests 3 | 4 | # Install dependencies 5 | RUN apt-get update -qq 6 | RUN apt-get install -qq -y software-properties-common 7 | RUN apt-get install -qq -y libsasl2-dev 8 | RUN add-apt-repository -y ppa:deadsnakes/ppa 9 | RUN apt-get update -qq 10 | RUN apt-get -qq install -y \ 11 | build-essential \ 12 | default-jre \ 13 | git \ 14 | libglib2.0-0 \ 15 | libsm6 \ 16 | libxrender1 \ 17 | python-pip \ 18 | python-software-properties \ 19 | python2.7 \ 20 | python2.7-dev \ 21 | python3-pip \ 22 | python3.6 \ 23 | python3.6-dev \ 24 | python3.6-venv \ 25 | virtualenv \ 26 | && rm -rf /var/lib/apt/lists/* 27 | 28 | # Temporarily add files needed for env setup. 29 | RUN mkdir /hops 30 | ADD setup.py /hops/ 31 | ADD README.rst /hops/ 32 | ADD hops /hops/hops 33 | 34 | # Set up Python3 environment 35 | RUN python3.6 -m pip install pip --upgrade 36 | RUN python3.6 -m pip install wheel 37 | RUN python3.6 -m venv /hops_venv3.6 38 | RUN /hops_venv3.6/bin/pip3.6 install -e /hops/[docs,tf,test,spark,plotting,pynvml,pyopenssl] # install dependencies 39 | RUN /hops_venv3.6/bin/pip3.6 uninstall -y hops # uninstall hops since we will use the latest source when running tests 40 | 41 | # Clean up temporary files and make room for mounted hops dir 42 | RUN rm -r /hops 43 | -------------------------------------------------------------------------------- /docker/Makefile: -------------------------------------------------------------------------------- 1 | all: rm-image build 2 | 3 | # Build the hops container 4 | # Note - the build process can be memory intensive due to 5 | # pip. If the install process is killed midway through, 6 | # rerun it and check `docker stats` during execution; you may 7 | # need to increase Docker's memory limits. 8 | 9 | build: Dockerfile 10 | cd .. && docker build . \ 11 | -f ./docker/Dockerfile \ 12 | --tag hops 13 | 14 | 15 | # Run a container to serve as a development workspace with 16 | # the local repo mounted into it. IMPORTANT: this gives 17 | # the container access to the host's filesystem! 18 | run: build 19 | docker run \ 20 | -dt \ 21 | -v `pwd`/..:/hops \ 22 | --name hops \ 23 | hops 24 | 25 | # Create a shell into a running container 26 | shell: 27 | docker exec -it hops /bin/bash 28 | 29 | # Stop and remove running containers 30 | clean: 31 | -docker stop hops 32 | -docker rm hops 33 | 34 | # Expunge old container image 35 | rm-image: 36 | -docker rmi hops 37 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Dockerfile 2 | 3 | This docker file is used to create a reproducible environment for running `hops-util-py` Unit tests for python3.6 and python 2.7. 4 | 5 | To build the docker image: 6 | 7 | ```bash 8 | cd docker # go to where Makefile is located 9 | # Build docker container and mount (you might need sudo), 10 | # this will fail if you already have a container named "hops" running 11 | # (if that is the case, just run `make shell` instead, or if you want the kill the existing container and re-build, 12 | # execute: `make clean`) 13 | make build run shell 14 | ``` 15 | The docker approach is recommended if you already have an existing SPARK/Hadoop installation in your environment to avoid conflicts ( for example if you want to run the tests from inside a VM where Hops is deployed). 16 | 17 | Also note that when you edit files inside your local machine the changes will automatically get reflected in the docker container as well since the directory is mounted there. 18 | 19 | To open up a shell in an already built container: 20 | 21 | ```bash 22 | cd docker # go to where Makefile is located 23 | make shell 24 | ``` 25 | 26 | To kill an existing container: 27 | 28 | ```bash 29 | cd docker # go to where Makefile is located 30 | make clean 31 | ``` -------------------------------------------------------------------------------- /docker/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # A simple shell script that is meant to be run inside a docker container for running hops-util-py unit tests 4 | # The script will take an argument that is the python version (supported versions currently are : 2.7 and 3.6) 5 | # the virtual env corresponding to the python version will be activated and then the unit tests will be run inside 6 | # that environment 7 | # example usage: 8 | # ./run.sh 2.7 9 | # ./run.sh 3.6 10 | 11 | PYTHON_VER=$1 12 | 13 | if [[ "${PYTHON_VER}" != "3.6" && "${PYTHON_VER}" != "2.7" ]] 14 | then 15 | echo "Invalid python version, supported versions are : 3.6 and 2.7" 16 | exit 1 17 | fi 18 | 19 | echo "Running Unit Tests with Python ${PYTHON_VER}" 20 | 21 | source /hops_venv${PYTHON_VER}/bin/activate 22 | cd /hops 23 | pip install -e . 24 | pytest -v hops $2 25 | exit 0 -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | # SOURCEDIR = source 9 | BUILDDIR = _build 10 | 11 | # User-friendly check for sphinx-build 12 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 13 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 14 | endif 15 | 16 | # Internal variables. 17 | PAPEROPT_a4 = -D latex_paper_size=a4 18 | PAPEROPT_letter = -D latex_paper_size=letter 19 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 20 | # the i18n builder cannot share the environment and doctrees with the others 21 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 22 | 23 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 24 | 25 | help: 26 | @echo "Please use \`make ' where is one of" 27 | @echo " html to make standalone HTML files" 28 | @echo " dirhtml to make HTML files named index.html in directories" 29 | @echo " singlehtml to make a single large HTML file" 30 | @echo " pickle to make pickle files" 31 | @echo " json to make JSON files" 32 | @echo " htmlhelp to make HTML files and a HTML help project" 33 | @echo " qthelp to make HTML files and a qthelp project" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | 50 | clean: 51 | rm -rf $(BUILDDIR)/* 52 | 53 | html: 54 | cp -rf ../imgs .;$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 55 | @echo 56 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 57 | 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | singlehtml: 64 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 65 | @echo 66 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 67 | 68 | pickle: 69 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 70 | @echo 71 | @echo "Build finished; now you can process the pickle files." 72 | 73 | json: 74 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 75 | @echo 76 | @echo "Build finished; now you can process the JSON files." 77 | 78 | htmlhelp: 79 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 80 | @echo 81 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 82 | ".hhp project file in $(BUILDDIR)/htmlhelp." 83 | 84 | qthelp: 85 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 86 | @echo 87 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 88 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 89 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ReadtheDocsTemplate.qhcp" 90 | @echo "To view the help file:" 91 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ReadtheDocsTemplate.qhc" 92 | 93 | devhelp: 94 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 95 | @echo 96 | @echo "Build finished." 97 | @echo "To view the help file:" 98 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ReadtheDocsTemplate" 99 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ReadtheDocsTemplate" 100 | @echo "# devhelp" 101 | 102 | epub: 103 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 104 | @echo 105 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 106 | 107 | latex: 108 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 109 | @echo 110 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 111 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 112 | "(use \`make latexpdf' here to do that automatically)." 113 | 114 | latexpdf: 115 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 116 | @echo "Running LaTeX files through pdflatex..." 117 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 118 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 119 | 120 | latexpdfja: 121 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 122 | @echo "Running LaTeX files through platex and dvipdfmx..." 123 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 124 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 125 | 126 | text: 127 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 128 | @echo 129 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 130 | 131 | man: 132 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 133 | @echo 134 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 135 | 136 | texinfo: 137 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 138 | @echo 139 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 140 | @echo "Run \`make' in that directory to run these through makeinfo" \ 141 | "(use \`make info' here to do that automatically)." 142 | 143 | info: 144 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 145 | @echo "Running Texinfo files through makeinfo..." 146 | make -C $(BUILDDIR)/texinfo info 147 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 148 | 149 | gettext: 150 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 151 | @echo 152 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 153 | 154 | changes: 155 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 156 | @echo 157 | @echo "The overview file is in $(BUILDDIR)/changes." 158 | 159 | linkcheck: 160 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 161 | @echo 162 | @echo "Link check complete; look for any errors in the above output " \ 163 | "or in $(BUILDDIR)/linkcheck/output.txt." 164 | 165 | doctest: 166 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 167 | @echo "Testing of doctests in the sources finished, look at the " \ 168 | "results in $(BUILDDIR)/doctest/output.txt." 169 | 170 | xml: 171 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 172 | @echo 173 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 174 | 175 | pseudoxml: 176 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 177 | @echo 178 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 179 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # hops-util-py documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Oct 9 15:29:47 2018. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | sys.path.insert(0, os.path.abspath('../')) 23 | from hops import version as hops_version 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.autodoc'] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix(es) of source filenames. 40 | # You can specify multiple suffix as a list of string: 41 | # 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = 'hops-util-py' 50 | copyright = '2019, Logical Clocks AB' 51 | author = 'Logical Clocks AB' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = str(hops_version.__version__) 59 | # The full version, including alpha/beta/rc tags. 60 | release = str(hops_version.__version__) 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This patterns also effect to html_static_path and html_extra_path 72 | exclude_patterns = [] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = 'sphinx' 76 | 77 | # If true, `todo` and `todoList` produce output, else they produce nothing. 78 | todo_include_todos = False 79 | 80 | 81 | # -- Options for HTML output ---------------------------------------------- 82 | 83 | # The theme to use for HTML and HTML Help pages. See the documentation for 84 | # a list of builtin themes. 85 | # 86 | #html_theme = 'alabaster' 87 | html_theme = "sphinx_rtd_theme" 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | # 93 | # html_theme_options = {} 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | # Custom sidebar templates, must be a dictionary that maps document names 101 | # to template names. 102 | # 103 | # This is required for the alabaster theme 104 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 105 | html_sidebars = { 106 | '**': [ 107 | 'about.html', 108 | 'navigation.html', 109 | 'relations.html', # needs 'show_related': True theme option to display 110 | 'searchbox.html', 111 | 'donate.html', 112 | ] 113 | } 114 | 115 | 116 | # -- Options for HTMLHelp output ------------------------------------------ 117 | 118 | # Output file base name for HTML help builder. 119 | htmlhelp_basename = 'hops-util-pydoc' 120 | 121 | 122 | # -- Options for LaTeX output --------------------------------------------- 123 | 124 | latex_elements = { 125 | # The paper size ('letterpaper' or 'a4paper'). 126 | # 127 | # 'papersize': 'letterpaper', 128 | 129 | # The font size ('10pt', '11pt' or '12pt'). 130 | # 131 | # 'pointsize': '10pt', 132 | 133 | # Additional stuff for the LaTeX preamble. 134 | # 135 | # 'preamble': '', 136 | 137 | # Latex figure (float) alignment 138 | # 139 | # 'figure_align': 'htbp', 140 | } 141 | 142 | # Grouping the document tree into LaTeX files. List of tuples 143 | # (source start file, target name, title, 144 | # author, documentclass [howto, manual, or own class]). 145 | latex_documents = [ 146 | (master_doc, 'hops-util-py.tex', 'hops-util-py Documentation', 147 | 'Logical Clocks AB', 'manual'), 148 | ] 149 | 150 | 151 | # -- Options for manual page output --------------------------------------- 152 | 153 | # One entry per manual page. List of tuples 154 | # (source start file, name, description, authors, manual section). 155 | man_pages = [ 156 | (master_doc, 'hops-util-py', 'hops-util-py Documentation', 157 | [author], 1) 158 | ] 159 | 160 | 161 | # -- Options for Texinfo output ------------------------------------------- 162 | 163 | # Grouping the document tree into Texinfo files. List of tuples 164 | # (source start file, target name, title, author, 165 | # dir menu entry, description, category) 166 | texinfo_documents = [ 167 | (master_doc, 'hops-util-py', 'hops-util-py Documentation', 168 | author, 'hops-util-py', 'Client library for interacting with Hopsworks services.', 169 | 'Miscellaneous'), 170 | ] 171 | 172 | autodoc_mock_imports = ["pyspark", "pydoop", "pydoop.hdfs", "jks", "IPython", "IPython.core.display", "tensorflow", "pynvml", "pyarrow", "pathlib", "pyhive", "numpy", "pandas", "boto3", "OpenSSL", "cryptography", "idna", "dns"] 173 | 174 | exclude_patterns = ["experiment_impl", "featurestore_impl"] 175 | 176 | autodoc_default_options = { 177 | 'members': None, 178 | 'member-order': 'bysource', 179 | 'special-members': '__init__', 180 | 'exclude-members': '__weakref__' 181 | } 182 | -------------------------------------------------------------------------------- /docs/hops.rst: -------------------------------------------------------------------------------- 1 | hops package 2 | ============ 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.dataset module 8 | ------------------- 9 | 10 | .. automodule:: hops.dataset 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.devices module 16 | ------------------- 17 | 18 | .. automodule:: hops.devices 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.experiment module 24 | ---------------------- 25 | 26 | .. automodule:: hops.experiment 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hops.model module 32 | ----------------- 33 | 34 | .. automodule:: hops.model 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hops.numpy_helper module 40 | ------------------------ 41 | 42 | .. automodule:: hops.numpy_helper 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | hops.pandas_helper module 48 | ------------------------- 49 | 50 | .. automodule:: hops.pandas_helper 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | hops.hdfs module 56 | ---------------- 57 | 58 | .. automodule:: hops.hdfs 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | hops.jobs module 64 | ---------------- 65 | 66 | .. automodule:: hops.jobs 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | hops.secret module 72 | ---------------- 73 | 74 | .. automodule:: hops.secret 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | hops.kafka module 80 | ----------------- 81 | 82 | .. automodule:: hops.kafka 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | hops.project module 88 | ------------------- 89 | 90 | .. automodule:: hops.project 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | hops.serving module 96 | ------------------- 97 | 98 | .. automodule:: hops.serving 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | hops.tensorboard module 104 | ----------------------- 105 | 106 | .. automodule:: hops.tensorboard 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | hops.tls module 112 | --------------- 113 | 114 | .. automodule:: hops.tls 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | hops.user module 120 | ------------------- 121 | 122 | .. automodule:: hops.user 123 | :members: 124 | :undoc-members: 125 | :show-inheritance: 126 | 127 | hops.util module 128 | ---------------- 129 | 130 | .. automodule:: hops.util 131 | :members: 132 | :undoc-members: 133 | :show-inheritance: 134 | 135 | hops.credentials_provider module 136 | ---------------- 137 | 138 | .. automodule:: hops.credentials_provider 139 | :members: 140 | :undoc-members: 141 | :show-inheritance: 142 | 143 | Module contents 144 | --------------- 145 | 146 | .. automodule:: hops 147 | :members: 148 | :undoc-members: 149 | :show-inheritance: 150 | -------------------------------------------------------------------------------- /docs/imgs/add-python-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/add-python-module.png -------------------------------------------------------------------------------- /docs/imgs/driver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/driver.png -------------------------------------------------------------------------------- /docs/imgs/executor-hdfs-log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/executor-hdfs-log.png -------------------------------------------------------------------------------- /docs/imgs/executor-stderr1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/executor-stderr1.png -------------------------------------------------------------------------------- /docs/imgs/executor-stderr2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/executor-stderr2.png -------------------------------------------------------------------------------- /docs/imgs/executor-stderr3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/executor-stderr3.png -------------------------------------------------------------------------------- /docs/imgs/executor-stderr4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/executor-stderr4.png -------------------------------------------------------------------------------- /docs/imgs/hopsml-pyspark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/docs/imgs/hopsml-pyspark.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. hops-util-py documentation master file, created by 2 | sphinx-quickstart on Tue Oct 9 15:29:47 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. include:: ../README.rst 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | :hidden: 11 | 12 | API 13 | LICENSE 14 | 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | -------------------------------------------------------------------------------- /docs/license.rst: -------------------------------------------------------------------------------- 1 | License 2 | ------- 3 | 4 | Apache License, v2.0. See `LICENSE `_. 5 | -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | hops 2 | ==== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | hops 8 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/source/hops.rst: -------------------------------------------------------------------------------- 1 | hops package 2 | ============ 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.devices module 8 | ------------------- 9 | 10 | .. automodule:: hops.devices 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.exceptions module 16 | ---------------------- 17 | 18 | .. automodule:: hops.exceptions 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.experiment module 24 | ---------------------- 25 | 26 | .. automodule:: hops.experiment 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hops.hdfs module 32 | ---------------- 33 | 34 | .. automodule:: hops.hdfs 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hops.hive module 40 | ---------------- 41 | 42 | .. automodule:: hops.hive 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | hops.jobs module 48 | ---------------- 49 | 50 | .. automodule:: hops.jobs 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | hops.secret module 56 | ---------------- 57 | 58 | .. automodule:: hops.secret 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | hops.kafka module 64 | ----------------- 65 | 66 | .. automodule:: hops.kafka 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | hops.numpy\_helper module 72 | ------------------------- 73 | 74 | .. automodule:: hops.numpy_helper 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | hops.pandas\_helper module 80 | -------------------------- 81 | 82 | .. automodule:: hops.pandas_helper 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | hops.serving module 88 | ------------------- 89 | 90 | .. automodule:: hops.serving 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | hops.tensorboard module 96 | ----------------------- 97 | 98 | .. automodule:: hops.tensorboard 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | hops.tls module 104 | --------------- 105 | 106 | .. automodule:: hops.tls 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | hops.util module 112 | ---------------- 113 | 114 | .. automodule:: hops.util 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: hops 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | hops 2 | ==== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | hops 8 | -------------------------------------------------------------------------------- /hops/__init__.py: -------------------------------------------------------------------------------- 1 | from .version import __version__ 2 | 3 | import logging 4 | import sys 5 | 6 | logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout) 7 | -------------------------------------------------------------------------------- /hops/credentials_provider.py: -------------------------------------------------------------------------------- 1 | """ 2 | AWS temporary credential provider. 3 | 4 | """ 5 | 6 | from hops import constants, util, hdfs 7 | from hops.exceptions import RestAPIError 8 | import os 9 | 10 | 11 | def assume_role(role_arn=None, role_session_name=None, duration_seconds=3600): 12 | """ 13 | Assume a role and sets the temporary credential to the spark context hadoop configuration and environment variables. 14 | 15 | Args: 16 | :role_arn: (string) the role arn to be assumed 17 | :role_session_name: (string) use to uniquely identify a session when the same role is assumed by different principals or for different reasons. 18 | :duration_seconds: (int) the duration of the session. Maximum session duration is 3600 seconds. 19 | 20 | >>> from hops.credentials_provider import assume_role 21 | >>> assume_role(role_arn="arn:aws:iam:::role/analyst") 22 | 23 | or 24 | 25 | >>> assume_role() # to assume the default role 26 | 27 | Returns: 28 | temporary credentials 29 | """ 30 | query = _query_string(role_arn, role_session_name, duration_seconds) 31 | method = constants.HTTP_CONFIG.HTTP_GET 32 | resource_url = _get_cloud_resource() + constants.REST_CONFIG.HOPSWORKS_AWS_CLOUD_SESSION_TOKEN_RESOURCE + query 33 | 34 | response = util.send_request(method, resource_url) 35 | json_content = _parse_response(response, resource_url) 36 | _set_spark_hadoop_conf(json_content) 37 | _set_envs(json_content) 38 | return json_content 39 | 40 | 41 | def get_roles(): 42 | """ 43 | Get all roles mapped to the current project 44 | 45 | >>> from hops.credentials_provider import get_roles 46 | >>> get_roles() 47 | 48 | Returns: 49 | A list of role arn 50 | """ 51 | json_content = _get_roles() 52 | items = json_content[constants.REST_CONFIG.JSON_ARRAY_ITEMS] 53 | roles = [] 54 | for role in items: 55 | roles.append(role[constants.REST_CONFIG.JSON_CLOUD_ROLE]) 56 | return roles 57 | 58 | 59 | def get_role(role_id="default"): 60 | """ 61 | Get a role arn mapped to the current project by id or if no id is supplied the default role is returned 62 | 63 | Args: 64 | :role_id: id of the role default 65 | 66 | >>> from hops.credentials_provider import get_role 67 | >>> get_role(id) 68 | or 69 | >>> get_role() # to get the default role 70 | 71 | Returns: 72 | A role arn 73 | """ 74 | json_content = _get_roles(role_id=role_id) 75 | return json_content[constants.REST_CONFIG.JSON_CLOUD_ROLE] 76 | 77 | 78 | def _set_spark_hadoop_conf(json_content): 79 | spark = None 80 | if constants.ENV_VARIABLES.SPARK_IS_DRIVER in os.environ: 81 | spark = util._find_spark() 82 | if spark is not None: 83 | sc = spark.sparkContext 84 | sc._jsc.hadoopConfiguration().set(constants.S3_CONFIG.S3_CREDENTIAL_PROVIDER_ENV, 85 | constants.S3_CONFIG.S3_TEMPORARY_CREDENTIAL_PROVIDER) 86 | sc._jsc.hadoopConfiguration().set(constants.S3_CONFIG.S3_ACCESS_KEY_ENV, 87 | json_content[constants.REST_CONFIG.JSON_ACCESS_KEY_ID]) 88 | sc._jsc.hadoopConfiguration().set(constants.S3_CONFIG.S3_SECRET_KEY_ENV, 89 | json_content[constants.REST_CONFIG.JSON_SECRET_KEY_ID]) 90 | sc._jsc.hadoopConfiguration().set(constants.S3_CONFIG.S3_SESSION_KEY_ENV, 91 | json_content[constants.REST_CONFIG.JSON_SESSION_TOKEN_ID]) 92 | 93 | 94 | def _set_envs(json_content): 95 | os.environ[constants.S3_CONFIG.AWS_ACCESS_KEY_ID_ENV] = json_content[constants.REST_CONFIG.JSON_ACCESS_KEY_ID] 96 | os.environ[constants.S3_CONFIG.AWS_SECRET_ACCESS_KEY_ENV] = json_content[constants.REST_CONFIG.JSON_SECRET_KEY_ID] 97 | os.environ[constants.S3_CONFIG.AWS_SESSION_TOKEN_ENV] = json_content[constants.REST_CONFIG.JSON_SESSION_TOKEN_ID] 98 | 99 | 100 | def _get_roles(role_id=None): 101 | by_id = "" 102 | if role_id: 103 | by_id = constants.DELIMITERS.SLASH_DELIMITER + str(role_id) 104 | method = constants.HTTP_CONFIG.HTTP_GET 105 | resource_url = _get_cloud_resource() + constants.REST_CONFIG.HOPSWORKS_CLOUD_ROLE_MAPPINGS_RESOURCE + by_id 106 | response = util.send_request(method, resource_url) 107 | return _parse_response(response, resource_url) 108 | 109 | 110 | def _parse_response(response, url): 111 | if response.ok: 112 | return response.json() 113 | else: 114 | error_code, error_msg, user_msg = util._parse_rest_error(response.json()) 115 | raise RestAPIError("Error calling {}. Got status: HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}" 116 | ", user msg: {}".format(url, response.status_code, response.reason, error_code, error_msg, 117 | user_msg)) 118 | 119 | 120 | def _query_string(role, role_session_name, duration_seconds): 121 | query = "" 122 | if role: 123 | query = constants.REST_CONFIG.HOPSWORKS_CLOUD_SESSION_TOKEN_RESOURCE_QUERY_ROLE + "=" + role 124 | if role_session_name: 125 | if query != "": 126 | query += "&" 127 | query += constants.REST_CONFIG.HOPSWORKS_CLOUD_SESSION_TOKEN_RESOURCE_QUERY_SESSION + "=" + role_session_name 128 | if duration_seconds != 3600 and duration_seconds > 0: 129 | if query != "": 130 | query += "&" 131 | query += constants.REST_CONFIG.HOPSWORKS_CLOUD_SESSION_TOKEN_RESOURCE_QUERY_SESSION_DURATION + "=" + \ 132 | str(duration_seconds) 133 | 134 | if query != "": 135 | query = "?" + query 136 | return query 137 | 138 | 139 | def _get_cloud_resource(): 140 | return constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + \ 141 | constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + \ 142 | constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 143 | constants.REST_CONFIG.HOPSWORKS_CLOUD_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER 144 | -------------------------------------------------------------------------------- /hops/devices.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Utility functions to retrieve information about available devices in the environment. 4 | 5 | """ 6 | import subprocess 7 | import time 8 | import threading 9 | import os 10 | from pynvml import * 11 | import fnmatch 12 | 13 | import logging 14 | log = logging.getLogger(__name__) 15 | 16 | def _get_gpu_info(): 17 | """ 18 | Get the gpu information 19 | 20 | Returns: 21 | 22 | """ 23 | 24 | if _count_nvidia_gpus() > 0: 25 | try: 26 | gpu_str = '\n------------------------------ Found GPU device ------------------------------\n' 27 | nvmlInit() 28 | device_count = int(nvmlDeviceGetCount()) 29 | for i in range(device_count): 30 | handle = nvmlDeviceGetHandleByIndex(i) 31 | name = nvmlDeviceGetName(handle).decode("utf-8") 32 | mem_info = nvmlDeviceGetMemoryInfo(handle) 33 | util = nvmlDeviceGetUtilizationRates(handle) 34 | gpu_str += '[Type: ' + name + ', Memory Usage: ' + str(int(mem_info.used/1000000)) + '/' + str(int(mem_info.total/1000000)) + ' (MB), Current utilization: ' + str(int(util.gpu)) + '%]\n' 35 | gpu_str += '-----------------------------------------------------------------------------------\n' 36 | return gpu_str 37 | except Exception as err: 38 | log.error(err) 39 | pass 40 | finally: 41 | try: 42 | nvmlShutdown() 43 | except: 44 | pass 45 | else: 46 | return 'Could not find any GPUs accessible for the container' 47 | 48 | def _get_nvidia_gpu_util(): 49 | """ 50 | 51 | Returns: 52 | 53 | """ 54 | try: 55 | gpu_str = '\n------------------------------ GPU usage information ------------------------------\n' 56 | nvmlInit() 57 | device_count = int(nvmlDeviceGetCount()) 58 | for i in range(device_count): 59 | handle = nvmlDeviceGetHandleByIndex(i) 60 | name = nvmlDeviceGetName(handle).decode("utf-8") 61 | mem_info = nvmlDeviceGetMemoryInfo(handle) 62 | util = nvmlDeviceGetUtilizationRates(handle) 63 | gpu_str += '[Type: ' + name + ', Memory Usage: ' + str(int(mem_info.used/1000000)) + '/' + str(int(mem_info.total/1000000)) + ' (MB), Current utilization: ' + str(int(util.gpu)) + '%]\n' 64 | gpu_str += '-----------------------------------------------------------------------------------\n' 65 | return gpu_str 66 | except Exception as err: 67 | log.error(err) 68 | pass 69 | finally: 70 | try: 71 | nvmlShutdown() 72 | except Exception as err: 73 | log.error(err) 74 | pass 75 | 76 | return 'No GPU utilization information available, failed to initialize NVML' 77 | 78 | def _print_periodic_gpu_utilization(): 79 | """ 80 | 81 | Returns: 82 | 83 | """ 84 | t = threading.currentThread() 85 | nvidia_gpu = _count_nvidia_gpus() 86 | while getattr(t, "do_run", True): 87 | time.sleep(10) 88 | if nvidia_gpu > 0: 89 | print(_get_nvidia_gpu_util()) 90 | 91 | def _count_nvidia_gpus(): 92 | try: 93 | if 'EXECUTOR_GPUS' in os.environ: 94 | return int(os.environ['EXECUTOR_GPUS']) 95 | nvmlInit() 96 | return int(nvmlDeviceGetCount()) 97 | except Exception as err: 98 | return 0 99 | finally: 100 | try: 101 | nvmlShutdown() 102 | except: 103 | pass 104 | 105 | def get_num_gpus(): 106 | """ Get the number of GPUs available in the environment and consequently by the application 107 | Assuming there is one GPU in the environment 108 | 109 | >>> from hops import devices 110 | >>> devices.get_num_gpus() 111 | >>> 1 112 | 113 | Returns: 114 | Number of GPUs available in the environment 115 | """ 116 | try: 117 | return _count_nvidia_gpus() 118 | except Exception as err: 119 | return 0 -------------------------------------------------------------------------------- /hops/elasticsearch.py: -------------------------------------------------------------------------------- 1 | """ 2 | A module for setting up Elastcisearch spark connector. 3 | """ 4 | 5 | from hops import constants, util, hdfs, tls 6 | from hops.exceptions import RestAPIError 7 | import os 8 | 9 | 10 | def _get_elasticsearch_url(): 11 | return os.environ[constants.ENV_VARIABLES.ELASTIC_ENDPOINT_ENV_VAR] 12 | 13 | 14 | def get_elasticsearch_index(index): 15 | """ 16 | Get the valid elasticsearch index for later use. This helper method prefix the index name with the project name. 17 | 18 | Args: 19 | :index: the elasticsearch index to interact with. 20 | 21 | Returns: 22 | A valid elasticsearch index name. 23 | """ 24 | return (hdfs.project_name() + "_" + index).lower() 25 | 26 | 27 | def get_elasticsearch_config(index): 28 | """ 29 | Get the required elasticsearch configuration to setup a connection using spark connector. 30 | 31 | Args: 32 | :index: the elasticsearch index to interact with. 33 | 34 | Returns: 35 | A dictionary with required configuration. 36 | """ 37 | config = { 38 | constants.ELASTICSEARCH_CONFIG.SSL_CONFIG: "true", 39 | constants.ELASTICSEARCH_CONFIG.NODES: _get_elasticsearch_url(), 40 | constants.ELASTICSEARCH_CONFIG.NODES_WAN_ONLY: "true", 41 | constants.ELASTICSEARCH_CONFIG.SSL_KEYSTORE_LOCATION: tls.get_key_store(), 42 | constants.ELASTICSEARCH_CONFIG.SSL_KEYSTORE_PASSWORD: tls.get_key_store_pwd(), 43 | constants.ELASTICSEARCH_CONFIG.SSL_TRUSTSTORE_LOCATION: tls.get_trust_store(), 44 | constants.ELASTICSEARCH_CONFIG.SSL_TRUSTSTORE_PASSWORD: tls.get_trust_store_pwd(), 45 | constants.ELASTICSEARCH_CONFIG.HTTP_AUTHORIZATION: get_authorization_token(), 46 | constants.ELASTICSEARCH_CONFIG.INDEX: get_elasticsearch_index(index) 47 | } 48 | return config 49 | 50 | 51 | def get_authorization_token(): 52 | """ 53 | Get the authorization token to interact with elasticsearch. 54 | 55 | Args: 56 | 57 | Returns: 58 | The authorization token to be used in http header. 59 | """ 60 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 61 | method = constants.HTTP_CONFIG.HTTP_GET 62 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 63 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 64 | constants.REST_CONFIG.HOPSWORKS_ELASTIC_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 65 | constants.REST_CONFIG.HOPSWORKS_ELASTIC_JWT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 66 | hdfs.project_id() 67 | response = util.send_request(method, resource_url, headers=headers) 68 | response_object = response.json() 69 | if response.status_code >= 400: 70 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 71 | raise RestAPIError("Could not get authorization token for elastic (url: {}), server response: \n " \ 72 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 73 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 74 | 75 | return "Bearer " + response_object["token"] 76 | -------------------------------------------------------------------------------- /hops/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Common Exceptions thrown by the hops library 3 | """ 4 | 5 | class RestAPIError(Exception): 6 | """This exception will be raised if there is an error response from a REST API call to Hopsworks""" 7 | 8 | class UnkownSecretStorageError(Exception): 9 | """This exception will be raised if an unused secrets storage is passed as a parameter""" 10 | 11 | class APIKeyFileNotFound(Exception): 12 | """This exception will be raised if the file with the API Key is not found in the local filesystem""" -------------------------------------------------------------------------------- /hops/experiment_impl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/hops/experiment_impl/__init__.py -------------------------------------------------------------------------------- /hops/experiment_impl/distribute/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/hops/experiment_impl/distribute/__init__.py -------------------------------------------------------------------------------- /hops/experiment_impl/distribute/mirrored.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions to retrieve information about available services and setting up security for the Hops platform. 3 | 4 | These utils facilitates development by hiding complexity for programs interacting with Hops services. 5 | """ 6 | 7 | import os 8 | from hops import devices, tensorboard, hdfs 9 | from hops.experiment_impl.util import experiment_utils 10 | from hops import util 11 | 12 | import threading 13 | import time 14 | import socket 15 | import json 16 | 17 | from . import mirrored_reservation 18 | 19 | 20 | def _run(sc, train_fn, run_id, local_logdir=False, name="no-name", evaluator=False): 21 | """ 22 | 23 | Args: 24 | sc: 25 | train_fn: 26 | local_logdir: 27 | name: 28 | 29 | Returns: 30 | 31 | """ 32 | app_id = str(sc.applicationId) 33 | 34 | num_executions = util.num_executors() 35 | 36 | #Each TF task should be run on 1 executor 37 | nodeRDD = sc.parallelize(range(num_executions), num_executions) 38 | 39 | #Make SparkUI intuitive by grouping jobs 40 | sc.setJobGroup(os.environ['ML_ID'], "{} | MirroredStrategy - Distributed Training".format(name)) 41 | 42 | server = mirrored_reservation.Server(num_executions) 43 | server_addr = server.start() 44 | 45 | #Force execution on executor, since GPU is located on executor 46 | nodeRDD.foreachPartition(_prepare_func(app_id, run_id, train_fn, local_logdir, server_addr, evaluator, util.num_executors())) 47 | 48 | logdir = experiment_utils._get_logdir(app_id, run_id) 49 | 50 | print('Finished Experiment \n') 51 | 52 | path_to_return = logdir + '/.outputs.json' 53 | if hdfs.exists(path_to_return): 54 | with hdfs.open_file(path_to_return, flags="r") as fi: 55 | contents = fi.read() 56 | fi.close() 57 | return logdir, json.loads(contents) 58 | 59 | return logdir, None 60 | 61 | def _prepare_func(app_id, run_id, train_fn, local_logdir, server_addr, evaluator, num_executors): 62 | """ 63 | 64 | Args: 65 | app_id: 66 | run_id: 67 | train_fn: 68 | local_logdir: 69 | server_addr: 70 | 71 | Returns: 72 | 73 | """ 74 | def _wrapper_fun(iter): 75 | """ 76 | 77 | Args: 78 | iter: 79 | 80 | Returns: 81 | 82 | """ 83 | 84 | for i in iter: 85 | executor_num = i 86 | 87 | experiment_utils._set_ml_id(app_id, run_id) 88 | 89 | t = threading.Thread(target=devices._print_periodic_gpu_utilization) 90 | if devices.get_num_gpus() > 0: 91 | t.start() 92 | 93 | is_chief = False 94 | logdir = None 95 | tb_hdfs_path = None 96 | try: 97 | host = experiment_utils._get_ip_address() 98 | 99 | tmp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 100 | tmp_socket.bind(('', 0)) 101 | port = tmp_socket.getsockname()[1] 102 | 103 | client = mirrored_reservation.Client(server_addr) 104 | host_port = host + ":" + str(port) 105 | 106 | client.register({"worker": host_port, "index": executor_num}) 107 | cluster = client.await_reservations() 108 | tmp_socket.close() 109 | client.close() 110 | 111 | task_index = experiment_utils._find_index(host_port, cluster) 112 | 113 | if task_index == -1: 114 | cluster["task"] = {"type": "chief", "index": 0} 115 | else: 116 | cluster["task"] = {"type": "worker", "index": task_index} 117 | 118 | evaluator_node = None 119 | if evaluator: 120 | last_worker_index = len(cluster["cluster"]["worker"])-1 121 | evaluator_node = cluster["cluster"]["worker"][last_worker_index] 122 | cluster["cluster"]["evaluator"] = [evaluator_node] 123 | del cluster["cluster"]["worker"][last_worker_index] 124 | if evaluator_node == host_port: 125 | cluster["task"] = {"type": "evaluator", "index": 0} 126 | 127 | print('TF_CONFIG: {} '.format(cluster)) 128 | 129 | if num_executors > 1: 130 | os.environ["TF_CONFIG"] = json.dumps(cluster) 131 | 132 | is_chief = (cluster["task"]["type"] == "chief") 133 | 134 | logfile = experiment_utils._init_logger(experiment_utils._get_logdir(app_id, run_id), role=cluster["task"]["type"], index=cluster["task"]["index"]) 135 | 136 | dist_logdir = experiment_utils._get_logdir(app_id, run_id) + '/logdir' 137 | if is_chief: 138 | hdfs.mkdir(dist_logdir) 139 | tensorboard._register(dist_logdir, experiment_utils._get_logdir(app_id, run_id), executor_num, local_logdir=local_logdir) 140 | else: 141 | tensorboard.events_logdir = dist_logdir 142 | 143 | print(devices._get_gpu_info()) 144 | print('-------------------------------------------------------') 145 | print('Started running task') 146 | task_start = time.time() 147 | retval = train_fn() 148 | 149 | if is_chief: 150 | experiment_utils._handle_return_simple(retval, experiment_utils._get_logdir(app_id, run_id), logfile) 151 | 152 | task_end = time.time() 153 | time_str = 'Finished task - took ' + experiment_utils._time_diff(task_start, task_end) 154 | print(time_str) 155 | print('-------------------------------------------------------') 156 | except: 157 | raise 158 | finally: 159 | experiment_utils._cleanup(tensorboard, t) 160 | 161 | return _wrapper_fun -------------------------------------------------------------------------------- /hops/experiment_impl/distribute/mirrored_reservation.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import nested_scopes 4 | from __future__ import print_function 5 | 6 | import logging 7 | import pickle 8 | import select 9 | import socket 10 | import struct 11 | import threading 12 | import time 13 | 14 | from hops import util 15 | from hops.experiment_impl.util import experiment_utils 16 | 17 | MAX_RETRIES = 3 18 | BUFSIZE = 1024*2 19 | 20 | class Reservations: 21 | """ 22 | Thread-safe store for node reservations. 23 | """ 24 | 25 | def __init__(self, required): 26 | self.required = required 27 | self.lock = threading.RLock() 28 | self.reservations = {"cluster": {"worker": [None] * required}} 29 | self.check_done = False 30 | 31 | def add(self, meta): 32 | """ 33 | Add a reservation. 34 | 35 | Args: 36 | :meta: a dictonary of metadata about a node 37 | """ 38 | with self.lock: 39 | self.reservations["cluster"]["worker"][meta["index"]] = meta["worker"] 40 | if self.remaining() == 0: 41 | #Sort the cluster_spec based on ip so adjacent workers end up on same machine 42 | self.reservations["cluster"]["worker"].sort(key=lambda x:str(x.split(':')[0])) 43 | chief = self.reservations["cluster"]["worker"][0] 44 | self.reservations["cluster"]["chief"] = [chief] 45 | del self.reservations["cluster"]["worker"][0] 46 | self.check_done = True 47 | 48 | def done(self): 49 | """Returns True if the ``required`` number of reservations have been fulfilled.""" 50 | with self.lock: 51 | return self.check_done 52 | 53 | def get(self): 54 | """Get the list of current reservations.""" 55 | with self.lock: 56 | logging.debug("Returning reservations array {0}".format(self.reservations)) 57 | return self.reservations 58 | 59 | def remaining(self): 60 | """Get a count of remaining/unfulfilled reservations.""" 61 | with self.lock: 62 | num_registered = 0 63 | for entry in self.reservations["cluster"]["worker"]: 64 | if entry != None: 65 | num_registered = num_registered + 1 66 | return self.required - num_registered 67 | 68 | class MessageSocket(object): 69 | """Abstract class w/ length-prefixed socket send/receive functions.""" 70 | 71 | def receive(self, sock): 72 | """ 73 | Receive a message on ``sock`` 74 | Args: 75 | sock: 76 | 77 | Returns: 78 | 79 | """ 80 | msg = None 81 | data = b'' 82 | recv_done = False 83 | recv_len = -1 84 | while not recv_done: 85 | buf = sock.recv(BUFSIZE) 86 | if buf is None or len(buf) == 0: 87 | raise Exception("socket closed") 88 | if recv_len == -1: 89 | recv_len = struct.unpack('>I', buf[:4])[0] 90 | data += buf[4:] 91 | recv_len -= len(data) 92 | else: 93 | data += buf 94 | recv_len -= len(buf) 95 | recv_done = (recv_len == 0) 96 | 97 | msg = pickle.loads(data) 98 | return msg 99 | 100 | def send(self, sock, msg): 101 | """ 102 | Send ``msg`` to destination ``sock`` 103 | Args: 104 | sock: 105 | msg: 106 | 107 | Returns: 108 | 109 | """ 110 | data = pickle.dumps(msg) 111 | buf = struct.pack('>I', len(data)) + data 112 | sock.sendall(buf) 113 | 114 | 115 | class Server(MessageSocket): 116 | """Simple socket server with length prefixed pickle messages""" 117 | reservations = None 118 | done = False 119 | 120 | def __init__(self, count): 121 | assert count > 0 122 | self.reservations = Reservations(count) 123 | 124 | def await_reservations(self, sc, status={}, timeout=600): 125 | """ 126 | Block until all reservations are received. 127 | 128 | Args: 129 | sc: 130 | status: 131 | timeout: 132 | 133 | Returns: 134 | 135 | """ 136 | timespent = 0 137 | while not self.reservations.done(): 138 | logging.info("waiting for {0} reservations".format(self.reservations.remaining())) 139 | # check status flags for any errors 140 | if 'error' in status: 141 | sc.cancelAllJobs() 142 | #sc.stop() 143 | #sys.exit(1) 144 | time.sleep(1) 145 | timespent += 1 146 | if (timespent > timeout): 147 | raise Exception("timed out waiting for reservations to complete") 148 | logging.info("all reservations completed") 149 | return self.reservations.get() 150 | 151 | def _handle_message(self, sock, msg): 152 | """ 153 | 154 | Args: 155 | sock: 156 | msg: 157 | 158 | Returns: 159 | 160 | """ 161 | logging.debug("received: {0}".format(msg)) 162 | msg_type = msg['type'] 163 | if msg_type == 'REG': 164 | self.reservations.add(msg['data']) 165 | MessageSocket.send(self, sock, 'OK') 166 | elif msg_type == 'QUERY': 167 | logging.info("waiting for {0} reservations".format(self.reservations.remaining())) 168 | MessageSocket.send(self, sock, self.reservations.done()) 169 | elif msg_type == 'QINFO': 170 | rinfo = self.reservations.get() 171 | MessageSocket.send(self, sock, rinfo) 172 | elif msg_type == 'STOP': 173 | logging.info("setting server.done") 174 | MessageSocket.send(self, sock, 'OK') 175 | self.done = True 176 | else: 177 | MessageSocket.send(self, sock, 'ERR') 178 | 179 | def start(self): 180 | """ 181 | Start listener in a background thread 182 | 183 | Returns: 184 | address of the Server as a tuple of (host, port) 185 | """ 186 | server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 187 | server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 188 | server_sock.bind(('', 0)) 189 | server_sock.listen(10) 190 | 191 | # hostname may not be resolvable but IP address probably will be 192 | host = experiment_utils._get_ip_address() 193 | port = server_sock.getsockname()[1] 194 | addr = (host,port) 195 | 196 | def _listen(self, sock): 197 | """ 198 | 199 | Args: 200 | self: 201 | sock: 202 | 203 | Returns: 204 | 205 | """ 206 | CONNECTIONS = [] 207 | CONNECTIONS.append(sock) 208 | 209 | while not self.done: 210 | read_socks, write_socks, err_socks = select.select(CONNECTIONS, [], [], 60) 211 | for sock in read_socks: 212 | if sock == server_sock: 213 | client_sock, client_addr = sock.accept() 214 | CONNECTIONS.append(client_sock) 215 | logging.debug("client connected from {0}".format(client_addr)) 216 | else: 217 | try: 218 | msg = self.receive(sock) 219 | self._handle_message(sock, msg) 220 | except Exception as e: 221 | logging.debug(e) 222 | sock.close() 223 | CONNECTIONS.remove(sock) 224 | 225 | server_sock.close() 226 | 227 | t = threading.Thread(target=_listen, args=(self, server_sock)) 228 | t.daemon = True 229 | t.start() 230 | 231 | return addr 232 | 233 | def stop(self): 234 | """Stop the Server's socket listener.""" 235 | self.done = True 236 | 237 | 238 | class Client(MessageSocket): 239 | """Client to register and await node reservations.""" 240 | sock = None #: socket to server TCP connection 241 | server_addr = None #: address of server 242 | 243 | def __init__(self, server_addr): 244 | """ 245 | 246 | Args: 247 | :server_addr: a tuple of (host, port) pointing to the Server. 248 | """ 249 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 250 | self.sock.connect(server_addr) 251 | self.server_addr = server_addr 252 | logging.info("connected to server at {0}".format(server_addr)) 253 | 254 | def _request(self, msg_type, msg_data=None): 255 | """ 256 | Helper function to wrap msg w/ msg_type. 257 | 258 | Args: 259 | msg_type: 260 | msg_data: 261 | 262 | Returns: 263 | 264 | """ 265 | msg = {} 266 | msg['type'] = msg_type 267 | if msg_data or ((msg_data == True) or (msg_data == False)): 268 | msg['data'] = msg_data 269 | 270 | done = False 271 | tries = 0 272 | while not done and tries < MAX_RETRIES: 273 | try: 274 | MessageSocket.send(self, self.sock, msg) 275 | done = True 276 | except socket.error as e: 277 | tries += 1 278 | if tries >= MAX_RETRIES: 279 | raise 280 | print("Socket error: {}".format(e)) 281 | self.sock.close() 282 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 283 | self.sock.connect(self.server_addr) 284 | 285 | logging.debug("sent: {0}".format(msg)) 286 | resp = MessageSocket.receive(self, self.sock) 287 | logging.debug("received: {0}".format(resp)) 288 | return resp 289 | 290 | def close(self): 291 | """Close the client socket.""" 292 | self.sock.close() 293 | 294 | def register(self, reservation): 295 | """ 296 | Register ``reservation`` with server. 297 | 298 | Args: 299 | reservation: 300 | 301 | Returns: 302 | 303 | """ 304 | resp = self._request('REG', reservation) 305 | return resp 306 | 307 | def get_reservations(self): 308 | """Get current list of reservations.""" 309 | cluster_info = self._request('QINFO') 310 | return cluster_info 311 | 312 | def await_reservations(self): 313 | """Poll until all reservations completed, then return cluster_info.""" 314 | done = False 315 | while not done: 316 | done = self._request('QUERY') 317 | time.sleep(1) 318 | reservations = self.get_reservations() 319 | return reservations 320 | 321 | def request_stop(self): 322 | """Request server stop.""" 323 | resp = self._request('STOP') 324 | return resp 325 | -------------------------------------------------------------------------------- /hops/experiment_impl/distribute/mirrored_reservation_client.py: -------------------------------------------------------------------------------- 1 | from . import mirrored_reservation 2 | import sys 3 | 4 | if __name__ == "__main__": 5 | host = sys.argv[1] 6 | port = int(sys.argv[2]) 7 | addr = (host, port) 8 | client = mirrored_reservation.Client(addr) 9 | client.request_stop() 10 | client.close() 11 | -------------------------------------------------------------------------------- /hops/experiment_impl/distribute/parameter_server.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions to retrieve information about available services and setting up security for the Hops platform. 3 | 4 | These utils facilitates development by hiding complexity for programs interacting with Hops services. 5 | """ 6 | 7 | import os 8 | from hops import devices, tensorboard, hdfs 9 | from hops.experiment_impl.util import experiment_utils 10 | from hops import util 11 | 12 | import threading 13 | import time 14 | import socket 15 | import json 16 | 17 | from . import parameter_server_reservation 18 | 19 | def _run(sc, train_fn, run_id, local_logdir=False, name="no-name", evaluator=False): 20 | """ 21 | 22 | Args: 23 | sc: 24 | train_fn: 25 | local_logdir: 26 | name: 27 | 28 | Returns: 29 | 30 | """ 31 | app_id = str(sc.applicationId) 32 | 33 | num_executions = util.num_executors() 34 | 35 | #Each TF task should be run on 1 executor 36 | nodeRDD = sc.parallelize(range(num_executions), num_executions) 37 | 38 | #Make SparkUI intuitive by grouping jobs 39 | sc.setJobGroup(os.environ['ML_ID'], "{} | ParameterServerStrategy - Distributed Training".format(name)) 40 | 41 | server = parameter_server_reservation.Server(num_executions) 42 | 43 | server_addr = server.start() 44 | 45 | num_ps = util.num_param_servers() 46 | 47 | #Force execution on executor, since GPU is located on executor 48 | nodeRDD.foreachPartition(_prepare_func(app_id, run_id, train_fn, local_logdir, server_addr, num_ps, evaluator)) 49 | 50 | logdir = experiment_utils._get_logdir(app_id, run_id) 51 | 52 | print('Finished Experiment \n') 53 | 54 | path_to_return = logdir + '/.outputs.json' 55 | if hdfs.exists(path_to_return): 56 | with hdfs.open_file(path_to_return, flags="r") as fi: 57 | contents = fi.read() 58 | fi.close() 59 | return logdir, json.loads(contents) 60 | 61 | return logdir, None 62 | 63 | def _prepare_func(app_id, run_id, train_fn, local_logdir, server_addr, num_ps, evaluator): 64 | """ 65 | 66 | Args: 67 | app_id: 68 | run_id: 69 | map_fun: 70 | local_logdir: 71 | server_addr: 72 | num_ps: 73 | 74 | Returns: 75 | 76 | """ 77 | 78 | def _wrapper_fun(iter): 79 | """ 80 | 81 | Args: 82 | iter: 83 | 84 | Returns: 85 | 86 | """ 87 | 88 | for i in iter: 89 | executor_num = i 90 | 91 | 92 | experiment_utils._set_ml_id(app_id, run_id) 93 | 94 | t = threading.Thread(target=devices._print_periodic_gpu_utilization) 95 | if devices.get_num_gpus() > 0: 96 | t.start() 97 | 98 | role = None 99 | logdir = None 100 | tb_hdfs_path = None 101 | 102 | client = parameter_server_reservation.Client(server_addr) 103 | 104 | try: 105 | host = experiment_utils._get_ip_address() 106 | 107 | tmp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 108 | tmp_socket.bind(('', 0)) 109 | port = tmp_socket.getsockname()[1] 110 | host_port = host + ":" + str(port) 111 | 112 | exec_spec = {} 113 | if executor_num < num_ps: 114 | exec_spec["task_type"] = "ps" 115 | else: 116 | exec_spec["task_type"] = "worker" 117 | exec_spec["host_port"] = host_port 118 | 119 | client.register(exec_spec) 120 | 121 | cluster = client.await_reservations() 122 | 123 | tmp_socket.close() 124 | 125 | role, index = experiment_utils._find_task_and_index(host_port, cluster) 126 | 127 | cluster_spec = {} 128 | cluster_spec["cluster"] = cluster 129 | cluster_spec["task"] = {"type": role, "index": index} 130 | 131 | evaluator_node = None 132 | if evaluator: 133 | last_worker_index = len(cluster_spec["cluster"]["worker"])-1 134 | evaluator_node = cluster_spec["cluster"]["worker"][last_worker_index] 135 | cluster_spec["cluster"]["evaluator"] = [evaluator_node] 136 | del cluster_spec["cluster"]["worker"][last_worker_index] 137 | if evaluator_node == host_port: 138 | role = "evaluator" 139 | cluster_spec["task"] = {"type": "evaluator", "index": 0} 140 | 141 | print('TF_CONFIG: {} '.format(cluster_spec)) 142 | os.environ["TF_CONFIG"] = json.dumps(cluster_spec) 143 | 144 | logfile = experiment_utils._init_logger(experiment_utils._get_logdir(app_id, run_id), role=role, index=cluster_spec["task"]["index"]) 145 | 146 | dist_logdir = experiment_utils._get_logdir(app_id, run_id) + '/logdir' 147 | 148 | is_chief = (cluster_spec["task"]["type"] == "chief") 149 | if is_chief: 150 | hdfs.mkdir(dist_logdir) 151 | tensorboard._register(dist_logdir, experiment_utils._get_logdir(app_id, run_id), executor_num, local_logdir=local_logdir) 152 | else: 153 | tensorboard.events_logdir = dist_logdir 154 | 155 | print(devices._get_gpu_info()) 156 | print('-------------------------------------------------------') 157 | print('Started running task') 158 | task_start = time.time() 159 | 160 | retval=None 161 | if role == "ps": 162 | ps_thread = threading.Thread(target=lambda: train_fn()) 163 | ps_thread.start() 164 | client.await_all_workers_finished() 165 | else: 166 | retval = train_fn() 167 | 168 | if role == "chief": 169 | experiment_utils._handle_return_simple(retval, experiment_utils._get_logdir(app_id, run_id), logfile) 170 | 171 | task_end = time.time() 172 | time_str = 'Finished task - took ' + experiment_utils._time_diff(task_start, task_end) 173 | print(time_str) 174 | print('-------------------------------------------------------') 175 | except: 176 | raise 177 | finally: 178 | if role != "ps": 179 | client.register_worker_finished() 180 | client.close() 181 | experiment_utils._cleanup(tensorboard, t) 182 | return _wrapper_fun -------------------------------------------------------------------------------- /hops/experiment_impl/distribute/parameter_server_client.py: -------------------------------------------------------------------------------- 1 | from . import parameter_server_reservation 2 | import sys 3 | 4 | if __name__ == "__main__": 5 | host = sys.argv[1] 6 | port = int(sys.argv[2]) 7 | addr = (host, port) 8 | client = parameter_server_reservation.Client(addr) 9 | client.request_stop() 10 | client.close() 11 | -------------------------------------------------------------------------------- /hops/experiment_impl/distribute/parameter_server_reservation.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import nested_scopes 4 | from __future__ import print_function 5 | 6 | import logging 7 | import pickle 8 | import select 9 | import socket 10 | import struct 11 | import threading 12 | import time 13 | 14 | from hops import util 15 | from hops.experiment_impl.util import experiment_utils 16 | 17 | MAX_RETRIES = 3 18 | BUFSIZE = 1024*2 19 | 20 | class Reservations: 21 | """Thread-safe store for node reservations.""" 22 | 23 | def __init__(self, required): 24 | """ 25 | 26 | Args: 27 | required: 28 | """ 29 | self.required = required 30 | self.lock = threading.RLock() 31 | self.reservations = [] 32 | self.cluster_spec = {} 33 | self.check_done = False 34 | 35 | def add(self, meta): 36 | """Add a reservation. 37 | 38 | Args: 39 | :meta: a dictonary of metadata about a node 40 | """ 41 | with self.lock: 42 | self.reservations.append(meta) 43 | 44 | if self.remaining() == 0: 45 | 46 | cluster_spec = {"chief": [], "ps": [], "worker": []} 47 | added_chief=False 48 | for entry in self.reservations: 49 | if entry["task_type"] == "ps": 50 | cluster_spec["ps"].append(entry["host_port"]) 51 | elif added_chief == False and entry["task_type"] == "worker": 52 | cluster_spec["chief"].append(entry["host_port"]) 53 | added_chief = True 54 | else: 55 | cluster_spec["worker"].append(entry["host_port"]) 56 | 57 | self.cluster_spec = cluster_spec 58 | 59 | self.check_done = True 60 | 61 | def done(self): 62 | """Returns True if the ``required`` number of reservations have been fulfilled.""" 63 | with self.lock: 64 | return self.check_done 65 | 66 | def get(self): 67 | """Get the list of current reservations.""" 68 | with self.lock: 69 | return self.cluster_spec 70 | 71 | def remaining(self): 72 | """Get a count of remaining/unfulfilled reservations.""" 73 | with self.lock: 74 | num_registered = len(self.reservations) 75 | return self.required - num_registered 76 | 77 | 78 | class WorkerFinished: 79 | """Thread-safe store for node reservations.""" 80 | 81 | def __init__(self, required): 82 | """ 83 | 84 | Args: 85 | :required: expected number of nodes in the cluster. 86 | """ 87 | self.required = required 88 | self.lock = threading.RLock() 89 | self.finished = 0 90 | self.check_done = False 91 | 92 | def add(self): 93 | """Add a reservation. 94 | 95 | Args: 96 | :meta: a dictonary of metadata about a node 97 | """ 98 | with self.lock: 99 | self.finished = self.finished + 1 100 | 101 | if self.remaining() == 0: 102 | self.check_done = True 103 | 104 | def done(self): 105 | """Returns True if the ``required`` number of reservations have been fulfilled.""" 106 | with self.lock: 107 | return self.check_done 108 | 109 | def remaining(self): 110 | """Get a count of remaining/unfulfilled reservations.""" 111 | with self.lock: 112 | return self.required - self.finished 113 | 114 | class MessageSocket(object): 115 | """Abstract class w/ length-prefixed socket send/receive functions.""" 116 | 117 | def receive(self, sock): 118 | """ 119 | Receive a message on ``sock`` 120 | 121 | Args: 122 | sock: 123 | 124 | Returns: 125 | 126 | """ 127 | msg = None 128 | data = b'' 129 | recv_done = False 130 | recv_len = -1 131 | while not recv_done: 132 | buf = sock.recv(BUFSIZE) 133 | if buf is None or len(buf) == 0: 134 | raise Exception("socket closed") 135 | if recv_len == -1: 136 | recv_len = struct.unpack('>I', buf[:4])[0] 137 | data += buf[4:] 138 | recv_len -= len(data) 139 | else: 140 | data += buf 141 | recv_len -= len(buf) 142 | recv_done = (recv_len == 0) 143 | 144 | msg = pickle.loads(data) 145 | return msg 146 | 147 | def send(self, sock, msg): 148 | """ 149 | Send ``msg`` to destination ``sock``. 150 | 151 | Args: 152 | sock: 153 | msg: 154 | 155 | Returns: 156 | 157 | """ 158 | data = pickle.dumps(msg) 159 | buf = struct.pack('>I', len(data)) + data 160 | sock.sendall(buf) 161 | 162 | 163 | class Server(MessageSocket): 164 | """Simple socket server with length prefixed pickle messages""" 165 | reservations = None 166 | done = False 167 | 168 | def __init__(self, count): 169 | """ 170 | 171 | Args: 172 | count: 173 | """ 174 | assert count > 0 175 | self.reservations = Reservations(count) 176 | self.worker_finished = WorkerFinished(util.num_executors() - util.num_param_servers()) 177 | 178 | def await_reservations(self, sc, status={}, timeout=600): 179 | """ 180 | Block until all reservations are received. 181 | 182 | Args: 183 | sc: 184 | status: 185 | timeout: 186 | 187 | Returns: 188 | 189 | """ 190 | timespent = 0 191 | while not self.reservations.done(): 192 | logging.info("waiting for {0} reservations".format(self.reservations.remaining())) 193 | # check status flags for any errors 194 | if 'error' in status: 195 | sc.cancelAllJobs() 196 | #sc.stop() 197 | #sys.exit(1) 198 | time.sleep(1) 199 | timespent += 1 200 | if (timespent > timeout): 201 | raise Exception("timed out waiting for reservations to complete") 202 | logging.info("all reservations completed") 203 | return self.reservations.get() 204 | 205 | def _handle_message(self, sock, msg): 206 | """ 207 | 208 | Args: 209 | sock: 210 | msg: 211 | 212 | Returns: 213 | 214 | """ 215 | logging.debug("received: {0}".format(msg)) 216 | msg_type = msg['type'] 217 | if msg_type == 'REG': 218 | self.reservations.add(msg['data']) 219 | MessageSocket.send(self, sock, 'OK') 220 | elif msg_type == 'REG_DONE': 221 | self.worker_finished.add() 222 | MessageSocket.send(self, sock, 'OK') 223 | elif msg_type == 'QUERY': 224 | MessageSocket.send(self, sock, self.reservations.done()) 225 | elif msg_type == 'QUERY_DONE': 226 | MessageSocket.send(self, sock, self.worker_finished.done()) 227 | elif msg_type == 'QINFO': 228 | rinfo = self.reservations.get() 229 | MessageSocket.send(self, sock, rinfo) 230 | elif msg_type == 'STOP': 231 | logging.info("setting server.done") 232 | MessageSocket.send(self, sock, 'OK') 233 | self.done = True 234 | else: 235 | MessageSocket.send(self, sock, 'ERR') 236 | 237 | def start(self): 238 | """ 239 | Start listener in a background thread 240 | 241 | Returns: 242 | address of the Server as a tuple of (host, port) 243 | """ 244 | server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 245 | server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 246 | server_sock.bind(('', 0)) 247 | server_sock.listen(10) 248 | 249 | # hostname may not be resolvable but IP address probably will be 250 | host = experiment_utils._get_ip_address() 251 | port = server_sock.getsockname()[1] 252 | addr = (host,port) 253 | 254 | def _listen(self, sock): 255 | CONNECTIONS = [] 256 | CONNECTIONS.append(sock) 257 | 258 | while not self.done: 259 | read_socks, write_socks, err_socks = select.select(CONNECTIONS, [], [], 60) 260 | for sock in read_socks: 261 | if sock == server_sock: 262 | client_sock, client_addr = sock.accept() 263 | CONNECTIONS.append(client_sock) 264 | logging.debug("client connected from {0}".format(client_addr)) 265 | else: 266 | try: 267 | msg = self.receive(sock) 268 | self._handle_message(sock, msg) 269 | except Exception as e: 270 | logging.debug(e) 271 | sock.close() 272 | CONNECTIONS.remove(sock) 273 | 274 | server_sock.close() 275 | 276 | t = threading.Thread(target=_listen, args=(self, server_sock)) 277 | t.daemon = True 278 | t.start() 279 | 280 | return addr 281 | 282 | def stop(self): 283 | """Stop the Server's socket listener.""" 284 | self.done = True 285 | 286 | 287 | class Client(MessageSocket): 288 | """Client to register and await node reservations. 289 | 290 | Args: 291 | :server_addr: a tuple of (host, port) pointing to the Server. 292 | """ 293 | sock = None #: socket to server TCP connection 294 | server_addr = None #: address of server 295 | 296 | def __init__(self, server_addr): 297 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 298 | self.sock.connect(server_addr) 299 | self.server_addr = server_addr 300 | logging.info("connected to server at {0}".format(server_addr)) 301 | 302 | def _request(self, msg_type, msg_data=None): 303 | """Helper function to wrap msg w/ msg_type.""" 304 | msg = {} 305 | msg['type'] = msg_type 306 | if msg_data or ((msg_data == True) or (msg_data == False)): 307 | msg['data'] = msg_data 308 | 309 | done = False 310 | tries = 0 311 | while not done and tries < MAX_RETRIES: 312 | try: 313 | MessageSocket.send(self, self.sock, msg) 314 | done = True 315 | except socket.error as e: 316 | tries += 1 317 | if tries >= MAX_RETRIES: 318 | raise 319 | print("Socket error: {}".format(e)) 320 | self.sock.close() 321 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 322 | self.sock.connect(self.server_addr) 323 | 324 | logging.debug("sent: {0}".format(msg)) 325 | resp = MessageSocket.receive(self, self.sock) 326 | logging.debug("received: {0}".format(resp)) 327 | return resp 328 | 329 | def close(self): 330 | """Close the client socket.""" 331 | self.sock.close() 332 | 333 | def register(self, reservation): 334 | """ 335 | Register ``reservation`` with server. 336 | 337 | Args: 338 | reservation: 339 | 340 | Returns: 341 | 342 | """ 343 | resp = self._request('REG', reservation) 344 | return resp 345 | 346 | def register_worker_finished(self): 347 | """ 348 | Register ``worker as finished`` with server. 349 | 350 | Returns: 351 | 352 | """ 353 | resp = self._request('REG_DONE') 354 | return resp 355 | 356 | def await_all_workers_finished(self): 357 | """ 358 | Poll until all reservations completed, then return cluster_info. 359 | 360 | Returns: 361 | 362 | """ 363 | done = False 364 | while not done: 365 | done = self._request('QUERY_DONE') 366 | time.sleep(5) 367 | return True 368 | 369 | def get_reservations(self): 370 | """ 371 | Get current list of reservations. 372 | 373 | Returns: 374 | 375 | """ 376 | cluster_info = self._request('QINFO') 377 | return cluster_info 378 | 379 | def await_reservations(self): 380 | """Poll until all reservations completed, then return cluster_info.""" 381 | done = False 382 | while not done: 383 | done = self._request('QUERY') 384 | time.sleep(1) 385 | reservations = self.get_reservations() 386 | return reservations 387 | 388 | def request_stop(self): 389 | """Request server stop.""" 390 | resp = self._request('STOP') 391 | return resp 392 | -------------------------------------------------------------------------------- /hops/experiment_impl/launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | Simple experiment implementation 3 | """ 4 | 5 | from hops.experiment_impl.util import experiment_utils 6 | from hops import devices, tensorboard, hdfs 7 | 8 | import threading 9 | import time 10 | import json 11 | import os 12 | import six 13 | 14 | 15 | def _run(sc, train_fn, run_id, args_dict=None, local_logdir=False, name="no-name"): 16 | """ 17 | 18 | Args: 19 | sc: 20 | train_fn: 21 | args_dict: 22 | local_logdir: 23 | name: 24 | 25 | Returns: 26 | 27 | """ 28 | 29 | app_id = str(sc.applicationId) 30 | 31 | 32 | if args_dict == None: 33 | num_executions = 1 34 | else: 35 | arg_lists = list(args_dict.values()) 36 | currentLen = len(arg_lists[0]) 37 | for i in range(len(arg_lists)): 38 | if currentLen != len(arg_lists[i]): 39 | raise ValueError('Length of each function argument list must be equal') 40 | num_executions = len(arg_lists[i]) 41 | 42 | sc.setJobGroup(os.environ['ML_ID'], "{} | Launcher running experiment".format(name)) 43 | #Each TF task should be run on 1 executor 44 | nodeRDD = sc.parallelize(range(num_executions), num_executions) 45 | 46 | #Force execution on executor, since GPU is located on executor 47 | nodeRDD.foreachPartition(_prepare_func(app_id, run_id, train_fn, args_dict, local_logdir)) 48 | 49 | print('Finished Experiment \n') 50 | 51 | # For single run return .return if exists 52 | if args_dict == None: 53 | path_to_return = experiment_utils._get_logdir(app_id, run_id) + '/.outputs.json' 54 | if hdfs.exists(path_to_return): 55 | return_json = hdfs.load(path_to_return) 56 | return_dict = json.loads(return_json) 57 | return experiment_utils._get_logdir(app_id, run_id), return_dict 58 | else: 59 | return experiment_utils._get_logdir(app_id, run_id), None 60 | elif num_executions == 1: 61 | arg_count = six.get_function_code(train_fn).co_argcount 62 | arg_names = six.get_function_code(train_fn).co_varnames 63 | argIndex = 0 64 | param_string = '' 65 | while arg_count > 0: 66 | param_name = arg_names[argIndex] 67 | param_val = args_dict[param_name][0] 68 | param_string += str(param_name) + '=' + str(param_val) + '&' 69 | arg_count -= 1 70 | argIndex += 1 71 | param_string = param_string[:-1] 72 | path_to_return = experiment_utils._get_logdir(app_id, run_id) + '/' + param_string + '/.outputs.json' 73 | if hdfs.exists(path_to_return): 74 | return_json = hdfs.load(path_to_return) 75 | return_dict = json.loads(return_json) 76 | return experiment_utils._get_logdir(app_id, run_id), return_dict 77 | else: 78 | return experiment_utils._get_logdir(app_id, run_id), None 79 | else: 80 | return experiment_utils._get_logdir(app_id, run_id), None 81 | 82 | #Helper to put Spark required parameter iter in function signature 83 | def _prepare_func(app_id, run_id, train_fn, args_dict, local_logdir): 84 | """ 85 | 86 | Args: 87 | app_id: 88 | run_id: 89 | train_fn: 90 | args_dict: 91 | local_logdir: 92 | 93 | Returns: 94 | 95 | """ 96 | def _wrapper_fun(iter): 97 | """ 98 | 99 | Args: 100 | iter: 101 | 102 | Returns: 103 | 104 | """ 105 | 106 | for i in iter: 107 | executor_num = i 108 | 109 | experiment_utils._set_ml_id(app_id, run_id) 110 | 111 | tb_hdfs_path = '' 112 | 113 | hdfs_exec_logdir = experiment_utils._get_logdir(app_id, run_id) 114 | 115 | t = threading.Thread(target=devices._print_periodic_gpu_utilization) 116 | if devices.get_num_gpus() > 0: 117 | t.start() 118 | 119 | try: 120 | #Arguments 121 | if args_dict: 122 | param_string, params, args = experiment_utils.build_parameters(train_fn, executor_num, args_dict) 123 | hdfs_exec_logdir, hdfs_appid_logdir = experiment_utils._create_experiment_subdirectories(app_id, run_id, param_string, 'grid_search', params=params) 124 | logfile = experiment_utils._init_logger(hdfs_exec_logdir) 125 | tb_hdfs_path, tb_pid = tensorboard._register(hdfs_exec_logdir, hdfs_appid_logdir, executor_num, local_logdir=local_logdir) 126 | print(devices._get_gpu_info()) 127 | print('-------------------------------------------------------') 128 | print('Started running task ' + param_string) 129 | task_start = time.time() 130 | retval = train_fn(*args) 131 | task_end = time.time() 132 | experiment_utils._handle_return_simple(retval, hdfs_exec_logdir, logfile) 133 | time_str = 'Finished task ' + param_string + ' - took ' + experiment_utils._time_diff(task_start, task_end) 134 | print(time_str) 135 | print('-------------------------------------------------------') 136 | else: 137 | tb_hdfs_path, tb_pid = tensorboard._register(hdfs_exec_logdir, hdfs_exec_logdir, executor_num, local_logdir=local_logdir) 138 | logfile = experiment_utils._init_logger(hdfs_exec_logdir) 139 | print(devices._get_gpu_info()) 140 | print('-------------------------------------------------------') 141 | print('Started running task') 142 | task_start = time.time() 143 | retval = train_fn() 144 | task_end = time.time() 145 | experiment_utils._handle_return_simple(retval, hdfs_exec_logdir, logfile) 146 | time_str = 'Finished task - took ' + experiment_utils._time_diff(task_start, task_end) 147 | print(time_str) 148 | print('-------------------------------------------------------') 149 | except: 150 | raise 151 | finally: 152 | experiment_utils._cleanup(tensorboard, t) 153 | 154 | return _wrapper_fun -------------------------------------------------------------------------------- /hops/experiment_impl/parallel/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/hops/experiment_impl/parallel/__init__.py -------------------------------------------------------------------------------- /hops/experiment_impl/parallel/grid_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | Gridsearch implementation 3 | """ 4 | 5 | from hops import hdfs, tensorboard, devices 6 | 7 | from hops.experiment_impl.util import experiment_utils 8 | from hops.experiment import Direction 9 | 10 | import threading 11 | import six 12 | import time 13 | import os 14 | 15 | def _run(sc, train_fn, run_id, args_dict, direction=Direction.MAX, local_logdir=False, name="no-name", optimization_key=None): 16 | """ 17 | Run the wrapper function with each hyperparameter combination as specified by the dictionary 18 | 19 | Args: 20 | sc: 21 | train_fn: 22 | args_dict: 23 | direction: 24 | local_logdir: 25 | name: 26 | 27 | Returns: 28 | 29 | """ 30 | app_id = str(sc.applicationId) 31 | num_executions = 1 32 | 33 | if direction.upper() != Direction.MAX and direction.upper() != Direction.MIN: 34 | raise ValueError('Invalid direction ' + direction + ', must be Direction.MAX or Direction.MIN') 35 | 36 | arg_lists = list(args_dict.values()) 37 | currentLen = len(arg_lists[0]) 38 | for i in range(len(arg_lists)): 39 | if currentLen != len(arg_lists[i]): 40 | raise ValueError('Length of each function argument list must be equal') 41 | num_executions = len(arg_lists[i]) 42 | 43 | #Each TF task should be run on 1 executor 44 | nodeRDD = sc.parallelize(range(num_executions), num_executions) 45 | 46 | #Make SparkUI intuitive by grouping jobs 47 | sc.setJobGroup(os.environ['ML_ID'], "{} | Grid Search".format(name)) 48 | 49 | #Force execution on executor, since GPU is located on executor 50 | nodeRDD.foreachPartition(_prepare_func(app_id, run_id, train_fn, args_dict, local_logdir, optimization_key)) 51 | 52 | arg_count = six.get_function_code(train_fn).co_argcount 53 | arg_names = six.get_function_code(train_fn).co_varnames 54 | exp_dir = experiment_utils._get_logdir(app_id, run_id) 55 | 56 | max_val, max_hp, min_val, min_hp, avg, max_return_dict, min_return_dict = experiment_utils._get_best(args_dict, num_executions, arg_names, arg_count, exp_dir, optimization_key) 57 | 58 | param_combination = "" 59 | best_val = "" 60 | return_dict = {} 61 | 62 | if direction.upper() == Direction.MAX: 63 | param_combination = max_hp 64 | best_val = str(max_val) 65 | return_dict = max_return_dict 66 | elif direction.upper() == Direction.MIN: 67 | param_combination = min_hp 68 | best_val = str(min_val) 69 | return_dict = min_return_dict 70 | 71 | print('Finished Experiment \n') 72 | 73 | best_dir = exp_dir + '/' + param_combination 74 | 75 | return best_dir, experiment_utils._get_params_dict(best_dir), best_val, return_dict 76 | 77 | def _prepare_func(app_id, run_id, train_fn, args_dict, local_logdir, optimization_key): 78 | """ 79 | 80 | Args: 81 | app_id: 82 | run_id: 83 | train_fn: 84 | args_dict: 85 | local_logdir: 86 | 87 | Returns: 88 | 89 | """ 90 | 91 | def _wrapper_fun(iter): 92 | """ 93 | 94 | Args: 95 | iter: 96 | 97 | Returns: 98 | 99 | """ 100 | 101 | for i in iter: 102 | executor_num = i 103 | 104 | experiment_utils._set_ml_id(app_id, run_id) 105 | 106 | tb_hdfs_path = '' 107 | hdfs_exec_logdir = '' 108 | 109 | t = threading.Thread(target=devices._print_periodic_gpu_utilization) 110 | if devices.get_num_gpus() > 0: 111 | t.start() 112 | 113 | try: 114 | #Arguments 115 | if args_dict: 116 | param_string, params, args = experiment_utils.build_parameters(train_fn, executor_num, args_dict) 117 | hdfs_exec_logdir, hdfs_appid_logdir = experiment_utils._create_experiment_subdirectories(app_id, run_id, param_string, 'grid_search', params=params) 118 | logfile = experiment_utils._init_logger(hdfs_exec_logdir) 119 | tb_hdfs_path, tb_pid = tensorboard._register(hdfs_exec_logdir, hdfs_appid_logdir, executor_num, local_logdir=local_logdir) 120 | print(devices._get_gpu_info()) 121 | print('-------------------------------------------------------') 122 | print('Started running task ' + param_string) 123 | task_start = time.time() 124 | retval = train_fn(*args) 125 | task_end = time.time() 126 | experiment_utils._handle_return(retval, hdfs_exec_logdir, optimization_key, logfile) 127 | time_str = 'Finished task ' + param_string + ' - took ' + experiment_utils._time_diff(task_start, task_end) 128 | print(time_str) 129 | print('Returning metric ' + str(retval)) 130 | print('-------------------------------------------------------') 131 | except: 132 | raise 133 | finally: 134 | experiment_utils._cleanup(tensorboard, t) 135 | 136 | return _wrapper_fun -------------------------------------------------------------------------------- /hops/experiment_impl/parallel/random_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | Random Search implementation 3 | """ 4 | 5 | from hops.experiment_impl.util import experiment_utils 6 | from hops import devices, tensorboard, hdfs 7 | from hops.experiment import Direction 8 | 9 | import threading 10 | import six 11 | import time 12 | import random 13 | import os 14 | 15 | def _run(sc, train_fn, run_id, args_dict, samples, direction=Direction.MAX, local_logdir=False, name="no-name", optimization_key=None): 16 | """ 17 | 18 | Args: 19 | sc: 20 | train_fn: 21 | args_dict: 22 | local_logdir: 23 | name: 24 | 25 | Returns: 26 | 27 | """ 28 | 29 | app_id = str(sc.applicationId) 30 | 31 | arg_lists = list(args_dict.values()) 32 | for i in range(len(arg_lists)): 33 | if len(arg_lists[i]) != 2: 34 | raise ValueError('Boundary list must contain exactly two elements, [lower_bound, upper_bound] for each hyperparameter') 35 | 36 | hp_names = args_dict.keys() 37 | 38 | random_dict = {} 39 | for hp in hp_names: 40 | lower_bound = args_dict[hp][0] 41 | upper_bound = args_dict[hp][1] 42 | 43 | assert lower_bound < upper_bound, "lower bound: " + str(lower_bound) + " must be less than upper bound: " + str(upper_bound) 44 | 45 | random_values = [] 46 | 47 | if type(lower_bound) is int and type(upper_bound) is int: 48 | for i in range(samples): 49 | random_values.append(random.randint(lower_bound, upper_bound)) 50 | elif (type(lower_bound) is float or type(lower_bound) is int) and (type(upper_bound) is float or type(upper_bound) is int): 51 | for i in range(samples): 52 | random_values.append(random.uniform(lower_bound, upper_bound)) 53 | else: 54 | raise ValueError('Only float and int is currently supported') 55 | 56 | random_dict[hp] = random_values 57 | 58 | random_dict, new_samples = _remove_duplicates(random_dict, samples) 59 | 60 | sc.setJobGroup(os.environ['ML_ID'], "{} | Random Search".format(name)) 61 | #Each TF task should be run on 1 executor 62 | nodeRDD = sc.parallelize(range(new_samples), new_samples) 63 | 64 | nodeRDD.foreachPartition(_prepare_func(app_id, run_id, train_fn, random_dict, local_logdir, optimization_key)) 65 | 66 | arg_count = six.get_function_code(train_fn).co_argcount 67 | arg_names = six.get_function_code(train_fn).co_varnames 68 | exp_dir = experiment_utils._get_logdir(app_id, run_id) 69 | 70 | max_val, max_hp, min_val, min_hp, avg, max_return_dict, min_return_dict = experiment_utils._get_best(random_dict, new_samples, arg_names, arg_count, exp_dir, optimization_key) 71 | 72 | param_combination = "" 73 | best_val = "" 74 | return_dict = {} 75 | 76 | if direction.upper() == Direction.MAX: 77 | param_combination = max_hp 78 | best_val = str(max_val) 79 | return_dict = max_return_dict 80 | elif direction.upper() == Direction.MIN: 81 | param_combination = min_hp 82 | best_val = str(min_val) 83 | return_dict = min_return_dict 84 | 85 | print('Finished Experiment \n') 86 | 87 | best_dir = exp_dir + '/' + param_combination 88 | 89 | return best_dir, experiment_utils._get_params_dict(best_dir), best_val, return_dict 90 | 91 | def _remove_duplicates(random_dict, samples): 92 | hp_names = random_dict.keys() 93 | concatenated_hp_combs_arr = [] 94 | for index in range(samples): 95 | separated_hp_comb = "" 96 | for hp in hp_names: 97 | separated_hp_comb = separated_hp_comb + str(random_dict[hp][index]) + "&" 98 | concatenated_hp_combs_arr.append(separated_hp_comb) 99 | 100 | entry_index = 0 101 | indices_to_skip = [] 102 | for entry in concatenated_hp_combs_arr: 103 | inner_index = 0 104 | for possible_dup_entry in concatenated_hp_combs_arr: 105 | if entry == possible_dup_entry and inner_index > entry_index: 106 | indices_to_skip.append(inner_index) 107 | inner_index = inner_index + 1 108 | entry_index = entry_index + 1 109 | indices_to_skip = list(set(indices_to_skip)) 110 | 111 | for hp in hp_names: 112 | index = 0 113 | pruned_duplicates_arr = [] 114 | for random_value in random_dict[hp]: 115 | if index not in indices_to_skip: 116 | pruned_duplicates_arr.append(random_value) 117 | index = index + 1 118 | random_dict[hp] = pruned_duplicates_arr 119 | 120 | return random_dict, samples - len(indices_to_skip) 121 | 122 | #Helper to put Spark required parameter iter in function signature 123 | def _prepare_func(app_id, run_id, train_fn, args_dict, local_logdir, optimization_key): 124 | """ 125 | 126 | Args: 127 | app_id: 128 | run_id: 129 | train_fn: 130 | args_dict: 131 | local_logdir: 132 | 133 | Returns: 134 | 135 | """ 136 | def _wrapper_fun(iter): 137 | """ 138 | 139 | Args: 140 | iter: 141 | 142 | Returns: 143 | 144 | """ 145 | 146 | for i in iter: 147 | executor_num = i 148 | 149 | 150 | experiment_utils._set_ml_id(app_id, run_id) 151 | 152 | tb_hdfs_path = '' 153 | hdfs_exec_logdir = '' 154 | 155 | t = threading.Thread(target=devices._print_periodic_gpu_utilization) 156 | if devices.get_num_gpus() > 0: 157 | t.start() 158 | 159 | try: 160 | #Arguments 161 | if args_dict: 162 | param_string, params, args = experiment_utils.build_parameters(train_fn, executor_num, args_dict) 163 | hdfs_exec_logdir, hdfs_appid_logdir = experiment_utils._create_experiment_subdirectories(app_id, run_id, param_string, 'random_search', params=params) 164 | logfile = experiment_utils._init_logger(hdfs_exec_logdir) 165 | tb_hdfs_path, tb_pid = tensorboard._register(hdfs_exec_logdir, hdfs_appid_logdir, executor_num, local_logdir=local_logdir) 166 | print(devices._get_gpu_info()) 167 | print('-------------------------------------------------------') 168 | print('Started running task ' + param_string) 169 | task_start = time.time() 170 | retval = train_fn(*args) 171 | task_end = time.time() 172 | experiment_utils._handle_return(retval, hdfs_exec_logdir, optimization_key, logfile) 173 | time_str = 'Finished task ' + param_string + ' - took ' + experiment_utils._time_diff(task_start, task_end) 174 | print(time_str) 175 | print('Returning metric ' + str(retval)) 176 | print('-------------------------------------------------------') 177 | except: 178 | raise 179 | finally: 180 | experiment_utils._cleanup(tensorboard, t) 181 | 182 | return _wrapper_fun -------------------------------------------------------------------------------- /hops/experiment_impl/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/hops/experiment_impl/util/__init__.py -------------------------------------------------------------------------------- /hops/featurestore.py: -------------------------------------------------------------------------------- 1 | """ 2 | A feature store client. This module exposes an API for interacting with feature stores in Hopsworks. 3 | """ 4 | 5 | import warnings 6 | 7 | 8 | def fs_formatwarning(message, category, filename, lineno, line=None): 9 | return "{}:{}: {}: {}\n".format(filename, lineno, category.__name__, message) 10 | 11 | 12 | class FeatureStoreDeprecationWarning(Warning): 13 | """A Warning to be raised when the featurestore module is imported.""" 14 | pass 15 | 16 | warnings.formatwarning = fs_formatwarning 17 | warnings.simplefilter("always", FeatureStoreDeprecationWarning) 18 | 19 | raise FeatureStoreDeprecationWarning("The `featurestore` module was deprecated with the" 20 | " introduction of the new `HSFS` client libraries to interact with the Hopsworks " 21 | " Feature Store. All functionality has been removed from this module.") 22 | -------------------------------------------------------------------------------- /hops/hive.py: -------------------------------------------------------------------------------- 1 | from pyhive import hive 2 | import os 3 | from . import tls 4 | 5 | def setup_hive_connection(database=None): 6 | """ 7 | export enviroment variable with Hive connection configuration 8 | so it can be used by ipython-sql and PyHive to establish a connection with Hive 9 | 10 | Args: 11 | :database: Name of the database to connect to 12 | 13 | """ 14 | 15 | if not ('HIVE_ENDPOINT' in os.environ and 'PROJECT_NAME' in os.environ) : 16 | raise EnvironmentError("HIVE_ENDPOINT or PROJECT_NAME is not exported." + 17 | "The Hive module is meant to be run only in the context of a Python kernel") 18 | 19 | connection_conf = { 20 | 'auth' : 'CERTIFICATES', 21 | 'keystore' : tls.get_key_store(), 22 | 'truststore' : tls.get_trust_store(), 23 | 'keystore_password' : tls._get_cert_pw() 24 | } 25 | 26 | database_name = database if database else os.environ['PROJECT_NAME'] 27 | 28 | os.environ['DATABASE_URL'] = "hive://" + os.environ['HIVE_ENDPOINT'] + "/" + database_name + '?' 29 | os.environ['DATABASE_URL'] = os.environ['DATABASE_URL'] + '&'.join(['%s=%s' % (key, value) for (key, value) in connection_conf.items()]) 30 | -------------------------------------------------------------------------------- /hops/jobs.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Utility functions to manage jobs in Hopsworks. 4 | 5 | """ 6 | from hops import constants, util, hdfs 7 | from hops.exceptions import RestAPIError 8 | import json 9 | 10 | 11 | def create_job(name, job_config): 12 | """ 13 | Create a job in Hopsworks 14 | 15 | Args: 16 | name: Name of the job to be created. 17 | job_config: A dictionary representing the job configuration 18 | 19 | Returns: 20 | HTTP(S)Connection 21 | """ 22 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 23 | job_config["appName"] = name 24 | method = constants.HTTP_CONFIG.HTTP_PUT 25 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 26 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 27 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 28 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 29 | constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 30 | name 31 | response = util.send_request(method, resource_url, data=json.dumps(job_config), headers=headers) 32 | response_object = response.json() 33 | if response.status_code >= 400: 34 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 35 | raise RestAPIError("Could not create job (url: {}), server response: \n " 36 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 37 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 38 | 39 | return response_object 40 | 41 | 42 | def _job_execution_action(name, args=None): 43 | """ 44 | Manages execution for the given job, start or stop. Submits an http request to the HOPSWORKS REST API. 45 | 46 | Returns: 47 | The job status. 48 | """ 49 | method = constants.HTTP_CONFIG.HTTP_PUT 50 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 51 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 52 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 53 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 54 | constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 55 | name + constants.DELIMITERS.SLASH_DELIMITER + \ 56 | constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE 57 | 58 | response = util.send_request(method, resource_url, args) 59 | response_object = response.json() 60 | if response.status_code >= 400: 61 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 62 | raise RestAPIError("Could not perform action on job's execution (url: {}), server response: \n " 63 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 64 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 65 | 66 | return response_object 67 | 68 | 69 | def start_job(name, args=None): 70 | """ 71 | Start an execution of the job. Only one execution can be active for a job. 72 | 73 | Returns: 74 | The job status. 75 | """ 76 | method = constants.HTTP_CONFIG.HTTP_POST 77 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 78 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 79 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 80 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 81 | constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 82 | name + constants.DELIMITERS.SLASH_DELIMITER + \ 83 | constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE 84 | 85 | response = util.send_request(method, resource_url, args) 86 | response_object = response.json() 87 | if response.status_code >= 400: 88 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 89 | raise RestAPIError("Could not perform action on job's execution (url: {}), server response: \n " 90 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 91 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 92 | 93 | return response_object 94 | 95 | 96 | def stop_job(name): 97 | """ 98 | Stop the current execution of the job. 99 | Returns: 100 | The job status. 101 | """ 102 | method = constants.HTTP_CONFIG.HTTP_PUT 103 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 104 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 105 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 106 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 107 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 108 | constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 109 | name + constants.DELIMITERS.SLASH_DELIMITER + \ 110 | constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 111 | "{EXECUTION_ID}" + constants.DELIMITERS.SLASH_DELIMITER + \ 112 | "status" 113 | 114 | status = {"status": "stopped"} 115 | # If no execution_id was provided, stop all active executions 116 | # Get all active execution IDs 117 | executions = get_executions(name, 118 | "?filter_by=state:INITIALIZING,RUNNING,ACCEPTED,NEW,NEW_SAVING,SUBMITTED," 119 | "STARTING_APP_MASTER") 120 | responses = [] 121 | if executions['count'] > 0: 122 | for execution in executions['items']: 123 | responses.append(util.http(resource_url.replace("{EXECUTION_ID}", str(execution['id'])), headers, method, 124 | json.dumps(status))) 125 | return responses 126 | 127 | 128 | def get_executions(name, query=""): 129 | """ 130 | Get a list of the currently running executions for this job. 131 | Returns: 132 | The job status. 133 | """ 134 | method = constants.HTTP_CONFIG.HTTP_GET 135 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 136 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 137 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 138 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 139 | constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 140 | name + constants.DELIMITERS.SLASH_DELIMITER + \ 141 | constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + query 142 | response = util.send_request(method, resource_url) 143 | response_object = response.json() 144 | if response.status_code >= 500: 145 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 146 | raise RestAPIError("Could not get current job's execution (url: {}), server response: \n " 147 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 148 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 149 | if response.status_code >= 400: 150 | return None 151 | return response_object 152 | -------------------------------------------------------------------------------- /hops/kafka.py: -------------------------------------------------------------------------------- 1 | """ 2 | A module for setting up Kafka Brokers and Consumers on the Hops platform. It hides the complexity of 3 | configuring Kafka by providing utility methods such as: 4 | 5 | - `get_broker_endpoints()`. 6 | - `get_security_protocol()`. 7 | - `get_kafka_default_config()`. 8 | - etc. 9 | 10 | Using these utility functions you can setup Kafka with the Kafka client-library of your choice, e.g SparkStreaming or 11 | confluent-kafka-python. For example, assuming that you have created a topic called "test" on Hopsworks and that you 12 | have installed confluent-kafka-python inside your project's anaconda environment: 13 | 14 | >>> from hops import kafka 15 | >>> from confluent_kafka import Producer, Consumer 16 | >>> TOPIC_NAME = "test" 17 | >>> config = kafka.get_kafka_default_config() 18 | >>> producer = Producer(config) 19 | >>> consumer = Consumer(config) 20 | >>> consumer.subscribe(["test"]) 21 | >>> # wait a little while before executing the rest of the code (put it in a different Jupyter cell) 22 | >>> # so that the consumer get chance to subscribe (asynchronous call) 23 | >>> for i in range(0, 10): 24 | >>> producer.produce(TOPIC_NAME, "message {}".format(i), "key", callback=delivery_callback) 25 | >>> # Trigger the sending of all messages to the brokers, 10sec timeout 26 | >>> producer.flush(10) 27 | >>> for i in range(0, 10): 28 | >>> msg = consumer.poll(timeout=5.0) 29 | >>> if msg is not None: 30 | >>> print('Consumed Message: {} from topic: {}'.format(msg.value(), msg.topic())) 31 | >>> else: 32 | >>> print("Topic empty, timeout when trying to consume message") 33 | 34 | 35 | Similarly, you can define a pyspark kafka consumer as follows, using the spark session defined in variable `spark` 36 | 37 | >>> from hops import kafka 38 | >>> from hops import tls 39 | >>> TOPIC_NAME = "test" 40 | >>> df = spark \.format("kafka") 41 | >>> .option("kafka.bootstrap.servers", kafka.get_broker_endpoints()) 42 | >>> .option("kafka.ssl.truststore.location", tls.get_trust_store()) 43 | >>> .option("kafka.ssl.truststore.password", tls.get_key_store_pwd()) 44 | >>> .option("kafka.ssl.keystore.location", tls.get_key_store()) 45 | >>> .option("kafka.ssl.keystore.password", tls.get_key_store_pwd()) 46 | >>> .option("kafka.ssl.key.password", tls.get_trust_store_pwd()) 47 | >>> .option("subscribe", TOPIC_NAME) 48 | >>> .load() 49 | """ 50 | 51 | import os 52 | from hops import constants, tls, util, hdfs 53 | from hops.exceptions import RestAPIError 54 | import json 55 | import sys 56 | 57 | # for backwards compatibility 58 | try: 59 | from ast import literal_eval 60 | from io import BytesIO 61 | from avro.io import DatumReader, BinaryDecoder 62 | import avro.schema 63 | except: 64 | pass 65 | 66 | def get_broker_endpoints(): 67 | """ 68 | Get Kafka broker endpoints as a string with broker-endpoints "," separated 69 | 70 | Returns: 71 | a string with broker endpoints comma-separated 72 | """ 73 | return os.environ[constants.ENV_VARIABLES.KAFKA_BROKERS_ENV_VAR].replace("INTERNAL://","") 74 | 75 | 76 | def get_security_protocol(): 77 | """ 78 | Gets the security protocol used for communicating with Kafka brokers in a Hopsworks cluster 79 | 80 | Returns: 81 | the security protocol for communicating with Kafka brokers in a Hopsworks cluster 82 | """ 83 | return constants.KAFKA_SSL_CONFIG.SSL 84 | 85 | 86 | def get_broker_endpoints_list(): 87 | """ 88 | Get Kafka broker endpoints as a list 89 | 90 | Returns: 91 | a list with broker endpoint strings 92 | """ 93 | return get_broker_endpoints().split(",") 94 | 95 | 96 | def get_kafka_default_config(): 97 | """ 98 | Gets a default configuration for running secure Kafka on Hops 99 | 100 | Returns: 101 | dict with config_property --> value 102 | """ 103 | default_config = { 104 | constants.KAFKA_PRODUCER_CONFIG.BOOTSTRAP_SERVERS_CONFIG: get_broker_endpoints(), 105 | constants.KAFKA_SSL_CONFIG.SECURITY_PROTOCOL_CONFIG: get_security_protocol(), 106 | constants.KAFKA_SSL_CONFIG.SSL_CA_LOCATION_CONFIG: tls.get_ca_chain_location(), 107 | constants.KAFKA_SSL_CONFIG.SSL_CERTIFICATE_LOCATION_CONFIG: tls.get_client_certificate_location(), 108 | constants.KAFKA_SSL_CONFIG.SSL_PRIVATE_KEY_LOCATION_CONFIG: tls.get_client_key_location(), 109 | "group.id": "something" 110 | } 111 | return default_config 112 | 113 | 114 | def get_schema(topic): 115 | """ 116 | Gets the Avro schema for a particular Kafka topic. 117 | 118 | Args: 119 | :topic: Kafka topic name 120 | 121 | Returns: 122 | Avro schema as a string object in JSON format 123 | """ 124 | method = constants.HTTP_CONFIG.HTTP_GET 125 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 126 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 127 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 128 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 129 | constants.REST_CONFIG.HOPSWORKS_KAFKA_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 130 | constants.REST_CONFIG.HOPSWORKS_TOPICS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 131 | topic + constants.DELIMITERS.SLASH_DELIMITER + \ 132 | constants.REST_CONFIG.HOPSWORKS_SUBJECTS_RESOURCE 133 | response = util.send_request(method, resource_url) 134 | response_object = response.json() 135 | 136 | if response.status_code != 200: 137 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 138 | raise RestAPIError("Could not get Avro schema (url: {}), server response: \n " 139 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 140 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 141 | 142 | return response_object['schema'] 143 | 144 | 145 | def parse_avro_msg(msg, avro_schema): 146 | """ 147 | Parses an avro record using a specified avro schema 148 | 149 | Args: 150 | :msg: the avro message to parse 151 | :avro_schema: the avro schema 152 | 153 | Returns: 154 | The parsed/decoded message 155 | """ 156 | reader = DatumReader(avro_schema) 157 | message_bytes = BytesIO(msg) 158 | decoder = BinaryDecoder(message_bytes) 159 | return reader.read(decoder) 160 | 161 | 162 | def convert_json_schema_to_avro(json_schema): 163 | """ 164 | Parses a JSON kafka topic schema returned by Hopsworks REST API into an avro schema 165 | 166 | Args: 167 | :json_schema: the json schema to convert 168 | 169 | Returns: 170 | the avro schema 171 | """ 172 | 173 | return avro.schema.parse(json_schema) 174 | 175 | 176 | 177 | class KafkaTopicDTO(object): 178 | """ 179 | Represents a KafkaTopic in Hopsworks 180 | """ 181 | 182 | def __init__(self, kafka_topic_dto_json): 183 | """ 184 | Initialize the kafka topic from JSON payload returned by Hopsworks REST API 185 | 186 | Args: 187 | :kafka_topic_dto_json: JSON data about the kafka topic returned from Hopsworks REST API 188 | """ 189 | self.name = kafka_topic_dto_json[constants.REST_CONFIG.JSON_KAFKA_TOPIC_NAME] 190 | self.schema_version = kafka_topic_dto_json[constants.REST_CONFIG.JSON_KAFKA_TOPIC_SCHEMA_VERSION] 191 | -------------------------------------------------------------------------------- /hops/numpy_helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | API for reading/writing numpy arrays to/from HDFS 3 | """ 4 | import hops.hdfs as hdfs 5 | import numpy as np 6 | import os 7 | 8 | def load(hdfs_filename, **kwds): 9 | """ 10 | Reads a file from HDFS into a Numpy Array 11 | 12 | Args: 13 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS). 14 | :**kwds: You can add any additional args found in numpy.load(...) 15 | 16 | Returns: 17 | A numpy array 18 | 19 | Raises: 20 | IOError: If the file does not exist 21 | """ 22 | local_path = _copyHdfsToLocalOverwrite(hdfs_filename) 23 | return np.load(local_path, **kwds) 24 | 25 | 26 | def loadtxt(hdfs_filename, **kwds): 27 | """ 28 | Load data from a text file in HDFS into a Numpy Array. 29 | Each row in the text file must have the same number of values. 30 | 31 | Args: 32 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS). 33 | :**kwds: You can add any additional args found in numpy.loadtxt(...) 34 | 35 | Returns: 36 | A numpy array 37 | 38 | Raises: 39 | IOError: If the file does not exist 40 | """ 41 | local_path = _copyHdfsToLocalOverwrite(hdfs_filename) 42 | return np.loadtxt(local_path, **kwds) 43 | 44 | def genfromtxt(hdfs_filename, **kwds): 45 | """ 46 | Load data from a HDFS text file, with missing values handled as specified. 47 | Each line past the first skip_header lines is split at the delimiter character, and characters following the comments character are discarded. 48 | 49 | Args: 50 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS). 51 | :**kwds: You can add any additional args found in numpy.loadtxt(...) 52 | 53 | Returns: 54 | A numpy array 55 | 56 | Raises: 57 | IOError: If the file does not exist 58 | """ 59 | local_path = _copyHdfsToLocalOverwrite(hdfs_filename) 60 | return np.genfromtxt(local_path, **kwds) 61 | 62 | def fromregex(hdfs_filename, **kwds): 63 | """ 64 | Construct an array from a text file, using regular expression parsing. 65 | 66 | Args: 67 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS). 68 | :**kwds: You can add any additional args found in numpy.loadtxt(...) 69 | 70 | Returns: 71 | A numpy array 72 | 73 | Raises: 74 | IOError: If the file does not exist 75 | """ 76 | local_path = _copyHdfsToLocalOverwrite(hdfs_filename) 77 | return np.fromregex(local_path, **kwds) 78 | 79 | def fromfile(hdfs_filename, **kwds): 80 | """ 81 | Construct an array from data in a text or binary file. 82 | 83 | Args: 84 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS). 85 | :**kwds: You can add any additional args found in numpy.loadtxt(...) 86 | 87 | Returns: 88 | A numpy array 89 | 90 | Raises: 91 | IOError: If the file does not exist 92 | """ 93 | local_path = _copyHdfsToLocalOverwrite(hdfs_filename) 94 | return np.fromregex(local_path, **kwds) 95 | 96 | 97 | def memmap(hdfs_filename, **kwds): 98 | """ 99 | Create a memory-map to an array stored in a binary file on disk. 100 | 101 | Args: 102 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS). 103 | :**kwds: You can add any additional args found in numpy.loadtxt(...) 104 | 105 | Returns: 106 | A memmap with dtype and shape that matches the data. 107 | 108 | Raises: 109 | IOError: If the file does not exist 110 | """ 111 | local_path = _copyHdfsToLocalOverwrite(hdfs_filename) 112 | return np.fromregex(local_path, **kwds) 113 | 114 | 115 | def save(hdfs_filename, data): 116 | """ 117 | Saves a numpy array to a file in HDFS 118 | 119 | Args: 120 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 121 | :data: numpy array 122 | 123 | Raises: 124 | IOError: If the local file does not exist 125 | """ 126 | local_file = os.path.basename(hdfs_filename) 127 | np.save(local_file, data) 128 | _copyToHdfsOverwrite(hdfs_filename) 129 | 130 | def savez(hdfs_filename, *args, **kwds): 131 | """ 132 | Save several arrays into a single file in uncompressed .npz format in HDFS 133 | If arguments are passed in with no keywords, the corresponding variable names, in the .npz file, are 'arr_0', 'arr_1', etc. 134 | If keyword arguments are given, the corresponding variable names, in the .npz file will match the keyword names. 135 | 136 | Args: 137 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 138 | :args: Arguments, optional 139 | Arrays to save to the file. Since it is not possible for Python to know the names of the arrays outside savez, 140 | the arrays will be saved with names 'arr_0', 'arr_1', and so on. These arguments can be any expression. 141 | :kwds: Keyword arguments, optional 142 | Arrays to save to the file. Arrays will be saved in the file with the keyword names. :data: numpy array 143 | 144 | Returns: None 145 | 146 | Raises: 147 | IOError: If the local file does not exist 148 | """ 149 | local_file = os.path.basename(hdfs_filename) 150 | np.savez(local_file, *args, **kwds) 151 | _copyToHdfsOverwrite(hdfs_filename) 152 | 153 | def savez_compressed(hdfs_filename, *args, **kwds): 154 | """ 155 | Save several arrays into a single file in uncompressed .npz format in HDFS 156 | If arguments are passed in with no keywords, the corresponding variable names, in the .npz file, are 'arr_0', 'arr_1', etc. 157 | If keyword arguments are given, the corresponding variable names, in the .npz file will match the keyword names. 158 | 159 | Args: 160 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 161 | :*args: Arguments, optional 162 | Arrays to save to the file. Since it is not possible for Python to know the names of the arrays outside savez, 163 | the arrays will be saved with names 'arr_0', 'arr_1', and so on. These arguments can be any expression. 164 | :**kwds: Keyword arguments, optional 165 | Arrays to save to the file. Arrays will be saved in the file with the keyword names. :data: numpy array 166 | 167 | Returns: None 168 | 169 | Raises: 170 | IOError: If the local file does not exist 171 | """ 172 | local_file = os.path.basename(hdfs_filename) 173 | np.savez_compressed(local_file, *args, **kwds) 174 | _copyToHdfsOverwrite(hdfs_filename) 175 | 176 | 177 | def _copyHdfsToLocalOverwrite(hdfs_filename): 178 | hdfs_path = hdfs._expand_path(hdfs_filename) 179 | local_path = hdfs.copy_to_local(hdfs_path, overwrite=True) 180 | return local_path 181 | 182 | def _copyToHdfsOverwrite(hdfs_filename): 183 | local_file = os.path.basename(hdfs_filename) 184 | hdfs_path = hdfs._expand_path(hdfs_filename, exists=False) 185 | if local_file in hdfs_path: 186 | # copy_to_hdfs expects directory to copy to, excluding the file name 187 | hdfs_path = hdfs_path.replace(local_file, "") 188 | hdfs.copy_to_hdfs(local_file, hdfs_path, overwrite=True) 189 | -------------------------------------------------------------------------------- /hops/pandas_helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | API for opening csv files into Pandas from HDFS 3 | """ 4 | import hops.hdfs as hdfs 5 | import pandas as pd 6 | 7 | def read_csv(hdfs_filename, **kwds): 8 | """ 9 | Reads a comma-separated values (csv) file from HDFS into a Pandas DataFrame 10 | 11 | Args: 12 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 13 | :**kwds: You can add any additional args found in pandas.read_csv(...) 14 | 15 | Returns: 16 | A pandas dataframe 17 | 18 | Raises: 19 | IOError: If the file does not exist 20 | """ 21 | hdfs_path = hdfs._expand_path(hdfs_filename) 22 | h = hdfs.get_fs() 23 | with h.open_file(hdfs_path, "rt") as f: 24 | data = pd.read_csv(f, **kwds) 25 | return data 26 | 27 | def read_parquet(hdfs_filename, **kwds): 28 | """ 29 | Load a parquet object from a HDFS path, returning a DataFrame. 30 | 31 | Args: 32 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 33 | :**kwds: You can add any additional args found in pandas.read_csv(...) 34 | 35 | Returns: 36 | A pandas dataframe 37 | 38 | Raises: 39 | IOError: If the file does not exist 40 | """ 41 | hdfs_path = hdfs._expand_path(hdfs_filename) 42 | h = hdfs.get_fs() 43 | with h.open_file(hdfs_path, "rt") as f: 44 | data = pd.read_parquet(f, **kwds) 45 | return data 46 | 47 | def read_json(hdfs_filename, **kwds): 48 | """ 49 | Convert a JSON string to pandas object. 50 | 51 | Args: 52 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 53 | :**kwds: You can add any additional args found in pandas.read_csv(...) 54 | 55 | Returns: 56 | A pandas dataframe 57 | 58 | Raises: 59 | IOError: If the file does not exist 60 | """ 61 | hdfs_path = hdfs._expand_path(hdfs_filename) 62 | h = hdfs.get_fs() 63 | with h.open_file(hdfs_path, "rt") as f: 64 | data = pd.read_json(f, **kwds) 65 | return data 66 | 67 | def read_excel(hdfs_filename, **kwds): 68 | """ 69 | Retrieve pandas object stored in HDFS file, optionally based on where criteria 70 | 71 | 72 | 73 | Args: 74 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 75 | :**kwds: You can add any additional args found in pandas.read_csv(...) 76 | 77 | Returns: 78 | A pandas dataframe 79 | 80 | Raises: 81 | IOError: If the file does not exist 82 | """ 83 | hdfs_path = hdfs._expand_path(hdfs_filename) 84 | h = hdfs.get_fs() 85 | with h.open_file(hdfs_path, "rt") as f: 86 | data = pd.read_excel(f, **kwds) 87 | return data 88 | 89 | 90 | def write_csv(hdfs_filename, dataframe, **kwds): 91 | """ 92 | Writes a pandas dataframe to a comma-separated values (csv) text file in HDFS. Overwrites the file if it already exists 93 | 94 | Args: 95 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 96 | :dataframe: a Pandas dataframe 97 | :**kwds: You can add any additional args found in pandas.to_csv(...) 98 | 99 | Raises: 100 | IOError: If the file does not exist 101 | """ 102 | hdfs_path = hdfs._expand_path(hdfs_filename, exists=False) 103 | h = hdfs.get_fs() 104 | with h.open_file(hdfs_path, "wt") as f: 105 | dataframe.to_csv(f, **kwds) 106 | 107 | 108 | def write_parquet(hdfs_filename, dataframe, **kwds): 109 | """ 110 | Writes a pandas dataframe to a parquet file in HDFS. Overwrites the file if it already exists 111 | 112 | Args: 113 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 114 | :dataframe: a Pandas dataframe 115 | :**kwds: You can add any additional args found in pandas.to_parequet(...) 116 | 117 | Raises: 118 | IOError: If the file does not exist 119 | """ 120 | hdfs_path = hdfs._expand_path(hdfs_filename, exists=False) 121 | h = hdfs.get_fs() 122 | with h.open_file(hdfs_path, "wb") as f: 123 | dataframe.to_parquet(f, **kwds) 124 | 125 | 126 | def write_json(hdfs_filename, dataframe, **kwds): 127 | """ 128 | Writes a pandas dataframe to a JSON file in HDFS. Overwrites the file if it already exists 129 | 130 | Args: 131 | :hdfs_filename: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS) 132 | :dataframe: a Pandas dataframe 133 | :**kwds: You can add any additional args found in pandas.to_json(...) 134 | 135 | Raises: 136 | IOError: If the file does not exist 137 | """ 138 | hdfs_path = hdfs._expand_path(hdfs_filename, exists=False) 139 | h = hdfs.get_fs() 140 | with h.open_file(hdfs_path, "wt") as f: 141 | dataframe.to_json(f, **kwds) 142 | 143 | 144 | -------------------------------------------------------------------------------- /hops/project.py: -------------------------------------------------------------------------------- 1 | """ 2 | A module for connecting to and working with Hopsworks projects. 3 | 4 | Using the utility functions you can connect to a project of a particular Hopsworks instance which sets up all the 5 | required environment variables and configuration parameters. Then you can use moduels such as `dataset` to interact 6 | with particular services of a project. 7 | """ 8 | 9 | import os 10 | import json 11 | from hops import util, constants 12 | 13 | 14 | def connect(project, host=None, port=443, scheme="https", hostname_verification=False, 15 | api_key=None, 16 | region_name=constants.AWS.DEFAULT_REGION, 17 | secrets_store=constants.LOCAL.LOCAL_STORE, 18 | trust_store_path=None): 19 | """ 20 | Connect to a project of a Hopworks instance. Sets up API key and REST API endpoint. 21 | 22 | Example usage: 23 | 24 | >>> project.connect("dev_featurestore", "localhost", api_key="api_key_file") 25 | 26 | Args: 27 | :project_name: the name of the project to be used 28 | :host: the hostname of the Hopsworks cluster. If none specified, the library will attempt to the one set by the environment variable constants.ENV_VARIABLES.REST_ENDPOINT_END_VAR 29 | :port: the REST port of the Hopsworks cluster 30 | :scheme: the scheme to use for connection to the REST API. 31 | :hostname_verification: whether or not to verify Hopsworks' certificate - default True 32 | :api_key: path to a file containing an API key or the actual API key value. For secrets_store=local only. 33 | :region_name: The name of the AWS region in which the required secrets are stored 34 | :secrets_store: The secrets storage to be used. Secretsmanager or parameterstore for AWS, local otherwise. 35 | :trust_store_path: path to the file containing the Hopsworks certificates 36 | 37 | Returns: 38 | None 39 | """ 40 | 41 | util.connect(host, port, scheme, hostname_verification, api_key, region_name, secrets_store, trust_store_path) 42 | os.environ[constants.ENV_VARIABLES.HOPSWORKS_PROJECT_NAME_ENV_VAR] = project 43 | 44 | # Get projectId as we need for the REST endpoint 45 | project_info = get_project_info(project) 46 | project_id = str(project_info['projectId']) 47 | os.environ[constants.ENV_VARIABLES.HOPSWORKS_PROJECT_ID_ENV_VAR] = project_id 48 | 49 | 50 | def create(new_project, owner=None): 51 | """ 52 | Creates a project in Hopsworks. 53 | 54 | >>> from hops import util, project 55 | >>> new_project = {"projectName": "MyProject4", "description": "", "retentionPeriod": "", "status": 0, 56 | >>> "services": ["JOBS", "KAFKA", "JUPYTER", "HIVE", "SERVING", "FEATURESTORE", "AIRFLOW"]} 57 | >>> 58 | >>> util.connect("localhost", api_key="api_key_file") 59 | >>> project.create(new_project) 60 | 61 | Args: 62 | :new_project: A dictionary with the new project attributes. 63 | :owner: Create a project for another user (owner). Only admin user can use this option. 64 | 65 | Returns: 66 | JSON response 67 | 68 | Raises: 69 | :RestAPIError: if there was an error in the REST call to Hopsworks 70 | """ 71 | if owner is None: 72 | project_endpoint = constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 73 | "?projectName=" + new_project['projectName'] 74 | else: 75 | project_endpoint = constants.REST_CONFIG.HOPSWORKS_ADMIN_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 76 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + "s" + \ 77 | constants.DELIMITERS.SLASH_DELIMITER + "createas" 78 | new_project["owner"] = owner 79 | 80 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 81 | return util.http(constants.DELIMITERS.SLASH_DELIMITER + 82 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + 83 | project_endpoint, 84 | headers=headers, 85 | method=constants.HTTP_CONFIG.HTTP_POST, 86 | data=json.dumps(new_project)) 87 | 88 | 89 | def get_project_info(project_name): 90 | """ 91 | Makes a REST call to hopsworks to get all metadata of a project for the provided project. 92 | 93 | Args: 94 | :project_name: the name of the project 95 | 96 | Returns: 97 | JSON response 98 | See https://github.com/logicalclocks/hopsworks-ee/blob/master/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectDTO.java 99 | 100 | Raises: 101 | :RestAPIError: if there was an error in the REST call to Hopsworks 102 | """ 103 | return util.http(constants.DELIMITERS.SLASH_DELIMITER + 104 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + 105 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + 106 | constants.REST_CONFIG.HOPSWORKS_PROJECT_INFO_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + 107 | project_name) 108 | 109 | def get_project_info_as_shared(project_name): 110 | """ 111 | Makes a REST call to hopsworks to get all metadata of a project for the provided project. 112 | 113 | Args: 114 | :project_name: the name of the project 115 | 116 | Returns: 117 | JSON response 118 | See https://github.com/logicalclocks/hopsworks-ee/blob/master/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectDTO.java 119 | 120 | Raises: 121 | :RestAPIError: if there was an error in the REST call to Hopsworks 122 | """ 123 | return util.http(constants.DELIMITERS.SLASH_DELIMITER + 124 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + 125 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + 126 | constants.REST_CONFIG.HOPSWORKS_AS_SHARED + constants.DELIMITERS.SLASH_DELIMITER + 127 | constants.REST_CONFIG.HOPSWORKS_PROJECT_INFO_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + 128 | project_name) 129 | 130 | def project_id_as_shared(name=None): 131 | """ 132 | Get the Hopsworks project id from the project name. This endpoint can be used also for projects parents of shared datasets 133 | 134 | Args: 135 | :name: the name of the project, current project if none is supplied 136 | Returns: the Hopsworks project id 137 | 138 | """ 139 | if not name: 140 | return os.environ[constants.ENV_VARIABLES.HOPSWORKS_PROJECT_ID_ENV_VAR] 141 | 142 | project_info = get_project_info_as_shared(name) 143 | return str(project_info['projectId']) -------------------------------------------------------------------------------- /hops/secret.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Utility functions to manage secrets in Hopsworks. 4 | 5 | """ 6 | 7 | from hops import constants, util, project 8 | from hops.exceptions import RestAPIError 9 | import json 10 | 11 | def create_secret(name, secret, project_name=None): 12 | """ 13 | Create a secret 14 | 15 | Creating a secret for this user 16 | 17 | >>> from hops import secret 18 | >>> secret_token = 'DIOK4jmgFdwadjnDDW98' 19 | >>> secret.create_secret('my_secret', secret_token) 20 | 21 | Creating a secret and share it with all members of a project 22 | 23 | >>> from hops import secret 24 | >>> secret_token = 'DIOK4jmgFdwadjnDDW98' 25 | >>> secret.create_secret('my_secret', secret_token, project_name='someproject') 26 | 27 | Args: 28 | name: Name of the secret to create 29 | secret: Value of the secret 30 | project_name: Name of the project to share the secret with 31 | """ 32 | 33 | secret_config = {'name': name, 'secret': secret} 34 | 35 | if project_name is None: 36 | secret_config['visibility'] = "PRIVATE" 37 | else: 38 | scope_project = project.get_project_info(project_name) 39 | secret_config['scope'] = scope_project['projectId'] 40 | secret_config['visibility'] = "PROJECT" 41 | 42 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 43 | method = constants.HTTP_CONFIG.HTTP_POST 44 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 45 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 46 | constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 47 | constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE 48 | response = util.send_request(method, resource_url, data=json.dumps(secret_config), headers=headers) 49 | response_object = response.json() 50 | if response.status_code >= 400: 51 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 52 | raise RestAPIError("Could not create secret (url: {}), server response: \n " 53 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 54 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 55 | 56 | def get_secret(name, owner=None): 57 | """ 58 | Get a secret 59 | 60 | Get a secret for this user 61 | 62 | >>> from hops import secret 63 | >>> secret.get_secret('my_secret') 64 | 65 | Get a secret shared with this project by the secret owner 66 | 67 | >>> from hops import secret 68 | >>> secret.get_secret('shared_secret', owner='username') 69 | Args: 70 | name: Name of the secret to get 71 | owner: The username of the user that created the secret 72 | Returns: 73 | The secret 74 | """ 75 | if owner is None: 76 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 77 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 78 | constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 79 | constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 80 | name 81 | else: 82 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 83 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 84 | constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 85 | constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 86 | constants.REST_CONFIG.HOPSWORKS_SHARED + "?name=" + name + "&owner=" + owner 87 | 88 | method = constants.HTTP_CONFIG.HTTP_GET 89 | 90 | response = util.send_request(method, resource_url) 91 | response_object = response.json() 92 | if response.status_code >= 400: 93 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 94 | raise RestAPIError("Could not get secret (url: {}), server response: \n " 95 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 96 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 97 | 98 | return response_object['items'][0]['secret'] 99 | 100 | def delete_secret(name): 101 | """ 102 | 103 | Delete a secret for this user 104 | 105 | >>> from hops import secret 106 | >>> secret.delete_secret('my_secret') 107 | 108 | Args: 109 | name: Name of the secret to delete 110 | """ 111 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 112 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 113 | constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 114 | constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 115 | name 116 | 117 | method = constants.HTTP_CONFIG.HTTP_DELETE 118 | 119 | response = util.send_request(method, resource_url) 120 | if response.status_code >= 400: 121 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 122 | raise RestAPIError("Could not delete secret (url: {}), server response: \n " 123 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 124 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) -------------------------------------------------------------------------------- /hops/service_discovery.py: -------------------------------------------------------------------------------- 1 | import os 2 | import dns.resolver 3 | 4 | class ServiceDiscovery: 5 | 6 | @staticmethod 7 | def get_any_service(service_name): 8 | return ServiceDiscovery.get_service(service_name)[0] 9 | 10 | @staticmethod 11 | def get_service(service_name): 12 | service_fqdn = ServiceDiscovery.construct_service_fqdn(service_name) 13 | answer = dns.resolver.query(service_fqdn, 'SRV') 14 | return [(a.target.to_text(), a.port) for a in answer] 15 | 16 | @staticmethod 17 | def construct_service_fqdn(service_name): 18 | consul_domain = os.getenv('SERVICE_DISCOVERY_DOMAIN', "consul") 19 | if service_name.endswith('.'): 20 | return service_name + "service." + consul_domain 21 | return service_name + ".service." + consul_domain 22 | -------------------------------------------------------------------------------- /hops/tensorboard.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Utility functions to manage the lifecycle of TensorBoard and get the path to write TensorBoard events. 4 | 5 | """ 6 | import socket 7 | import subprocess 8 | import time 9 | import os 10 | from hops import hdfs as hopshdfs 11 | from hops.experiment_impl.util import experiment_utils 12 | from hops import util 13 | import shutil 14 | 15 | root_logdir_path = None 16 | events_logdir = None 17 | tb_pid = 0 18 | tb_url = None 19 | tb_port = None 20 | endpoint = None 21 | debugger_endpoint = None 22 | pypath = None 23 | tb_path = None 24 | local_logdir_path = None 25 | local_logdir_bool = False 26 | 27 | def _register(hdfs_exec_dir, endpoint_dir, exec_num, local_logdir=False): 28 | """ 29 | 30 | Args: 31 | hdfs_exec_dir: 32 | endpoint_dir: 33 | exec_num: 34 | local_logdir: 35 | 36 | Returns: 37 | 38 | """ 39 | global tb_pid 40 | 41 | if tb_pid != 0: 42 | subprocess.Popen(["kill", str(tb_pid)]) 43 | 44 | _reset_global() 45 | 46 | global events_logdir 47 | events_logdir = hdfs_exec_dir 48 | 49 | global local_logdir_bool 50 | local_logdir_bool = local_logdir 51 | 52 | 53 | if tb_pid == 0: 54 | global pypath 55 | pypath = os.getenv("PYSPARK_PYTHON") 56 | 57 | #find free port 58 | tb_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 59 | tb_socket.bind(('',0)) 60 | global tb_port 61 | tb_addr, tb_port = tb_socket.getsockname() 62 | 63 | global tb_path 64 | tb_path = experiment_utils._find_tensorboard() 65 | 66 | tb_socket.close() 67 | 68 | tb_env = _init_tb_env() 69 | 70 | global local_logdir_path 71 | if local_logdir: 72 | local_logdir_path = os.getcwd() + '/local_logdir' 73 | if os.path.exists(local_logdir_path): 74 | shutil.rmtree(local_logdir_path) 75 | os.makedirs(local_logdir_path) 76 | else: 77 | os.makedirs(local_logdir_path) 78 | 79 | local_logdir_path = local_logdir_path + '/' 80 | tb_proc = subprocess.Popen([pypath, tb_path, "--logdir=%s" % local_logdir_path, "--port=%d" % tb_port, "--host=%s" % "0.0.0.0"], 81 | env=tb_env, preexec_fn=util._on_executor_exit('SIGTERM')) 82 | else: 83 | tb_proc = subprocess.Popen([pypath, tb_path, "--logdir=%s" % events_logdir, "--port=%d" % tb_port, "--host=%s" % "0.0.0.0"], 84 | env=tb_env, preexec_fn=util._on_executor_exit('SIGTERM')) 85 | 86 | tb_pid = tb_proc.pid 87 | 88 | host = socket.getfqdn() 89 | global tb_url 90 | tb_url = "http://{0}:{1}".format(host, tb_port) 91 | global endpoint 92 | endpoint = endpoint_dir + "/TensorBoard.task" + str(exec_num) 93 | 94 | #dump tb host:port to hdfs 95 | hopshdfs.dump(tb_url, endpoint) 96 | 97 | return endpoint, tb_pid 98 | 99 | def logdir(): 100 | """ 101 | Get the TensorBoard logdir. This function should be called in your wrapper function for Experiment, Parallel Experiment or Distributed Training and passed as the 102 | logdir for TensorBoard. 103 | 104 | *Case 1: local_logdir=True*, then the logdir is on the local filesystem, otherwise it is in the folder for your experiment in your project in HDFS. Once the experiment is finished all the files that are present in the directory will be uploaded to tour experiment directory in the Experiments dataset. 105 | 106 | *Case 2: local_logdir=False*, then the logdir is in HDFS in your experiment directory in the Experiments dataset. 107 | 108 | Returns: 109 | The path to store files for your experiment. The content is also visualized in TensorBoard. 110 | """ 111 | 112 | global local_logdir_bool 113 | if local_logdir_bool: 114 | return local_logdir_path 115 | 116 | global events_logdir 117 | return events_logdir 118 | 119 | def interactive_debugger(): 120 | """ 121 | 122 | Returns: address for interactive debugger in TensorBoard 123 | 124 | 125 | """ 126 | global debugger_endpoint 127 | debugger_endpoint =_restart_debugging() 128 | return debugger_endpoint 129 | 130 | def non_interactive_debugger(): 131 | """ 132 | 133 | Returns: address for non-interactive debugger in TensorBoard 134 | 135 | """ 136 | global debugger_endpoint 137 | debugger_endpoint =_restart_debugging(interactive=False) 138 | return debugger_endpoint 139 | 140 | def _restart_debugging(interactive=True): 141 | """ 142 | 143 | Args: 144 | interactive: 145 | 146 | Returns: 147 | 148 | """ 149 | global tb_pid 150 | 151 | #Kill existing TB 152 | proc = subprocess.Popen(["kill", str(tb_pid)]) 153 | proc.wait() 154 | 155 | debugger_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 156 | debugger_socket.bind(('',0)) 157 | debugger_addr, debugger_port = debugger_socket.getfqdn() 158 | 159 | debugger_socket.close() 160 | 161 | tb_env = _init_tb_env() 162 | 163 | global pypath 164 | global tb_path 165 | global tb_port 166 | 167 | if interactive: 168 | tb_proc = subprocess.Popen([pypath, tb_path, "--logdir=%s" % logdir(), "--port=%d" % tb_port, "--debugger_port=%d" % debugger_port, "--host=%s" % "0.0.0.0"], 169 | env=tb_env, preexec_fn=util._on_executor_exit('SIGTERM')) 170 | tb_pid = tb_proc.pid 171 | 172 | if not interactive: 173 | tb_proc = subprocess.Popen([pypath, tb_path, "--logdir=%s" % logdir(), "--port=%d" % tb_port, "--debugger_data_server_grpc_port=%d" % debugger_port, "--host=%s" % "0.0.0.0"], 174 | env=tb_env, preexec_fn=util._on_executor_exit('SIGTERM')) 175 | tb_pid = tb_proc.pid 176 | 177 | time.sleep(2) 178 | 179 | return 'localhost:' + str(debugger_port) 180 | 181 | def _init_tb_env(): 182 | tb_env = os.environ.copy() 183 | tb_env['CUDA_VISIBLE_DEVICES'] = '' 184 | tb_env['HIP_VISIBLE_DEVICES'] = '-1' 185 | tb_env['LC_ALL'] = 'C' 186 | tb_env['TMPDIR'] = os.getcwd() 187 | return tb_env 188 | 189 | 190 | def _reset_global(): 191 | """ 192 | 193 | Returns: 194 | 195 | """ 196 | global root_logdir_path 197 | global events_logdir 198 | global tb_pid 199 | global tb_url 200 | global tb_port 201 | global endpoint 202 | global debugger_endpoint 203 | global pypath 204 | global tb_path 205 | global local_logdir_path 206 | global local_logdir_bool 207 | 208 | root_logdir_path = None 209 | events_logdir = None 210 | tb_pid = 0 211 | tb_url = None 212 | tb_port = None 213 | endpoint = None 214 | debugger_endpoint = None 215 | pypath = None 216 | tb_path = None 217 | local_logdir_path = None 218 | local_logdir_bool = False 219 | -------------------------------------------------------------------------------- /hops/tests/test_resources/attendances_features.csv: -------------------------------------------------------------------------------- 1 | team_id,average_attendance,sum_attendance 2 | 26,3271.934,65438.68 3 | 27,4074.8047,81496.09 4 | 6,19595.973,391919.47 5 | 16,6462.462,129249.24 6 | 20,7226.672,144533.44 7 | 40,3189.8455,63796.91 8 | 15,7076.029,141520.58 9 | 43,2823.996,56479.92 10 | 9,9405.213,188104.27 11 | 17,7118.376,142367.52 12 | 23,4964.6475,99292.945 13 | 39,2420.8076,48416.152 14 | 11,10853.007,217060.14 15 | 33,3094.6753,61893.504 16 | 18,4903.378,98067.56 17 | 4,22409.615,448192.3 18 | 21,5241.2993,104825.984 19 | 31,3587.5015,71750.03 20 | 34,2532.1638,50643.277 21 | 28,3397.8066,67956.13 22 | 44,2429.1638,48583.277 23 | 12,9313.054,186261.08 24 | 22,5582.53,111650.59 25 | 47,1995.5691,39911.383 26 | 1,92301.086,1846021.8 27 | 13,7429.8564,148597.12 28 | 3,32154.229,643084.56 29 | 48,1840.2683,36805.367 30 | 5,25713.953,514279.06 31 | 19,4593.92,91878.4 32 | 41,2049.1565,40983.13 33 | 37,2701.0522,54021.043 34 | 35,3244.3965,64887.93 35 | 8,9372.814,187456.28 36 | 49,1038.5237,20770.475 37 | 7,14111.81,282236.2 38 | 10,8980.543,179610.86 39 | 50,1673.24,33464.8 40 | 45,2095.7812,41915.625 41 | 38,2593.7607,51875.215 42 | 25,3585.4563,71709.125 43 | 24,4464.632,89292.63 44 | 29,5132.148,102642.96 45 | 32,2983.8452,59676.902 46 | 14,7088.685,141773.7 47 | 42,1787.7102,35754.203 48 | 2,35453.332,709066.6 49 | 30,3473.2007,69464.016 50 | 46,1940.3131,38806.26 51 | 36,2695.4463,53908.926 52 | -------------------------------------------------------------------------------- /hops/tests/test_resources/games_features.csv: -------------------------------------------------------------------------------- 1 | away_team_id,home_team_id,score 2 | 11,48,1 3 | 46,32,3 4 | 15,13,2 5 | 11,47,1 6 | 50,1,3 7 | 10,24,1 8 | 36,39,2 9 | 14,8,3 10 | 38,48,1 11 | 6,31,1 12 | 3,26,1 13 | 43,4,3 14 | 39,46,1 15 | 4,41,1 16 | 13,20,1 17 | 14,33,1 18 | 2,8,1 19 | 8,15,1 20 | 24,40,1 21 | 27,42,1 22 | 34,27,3 23 | 32,20,3 24 | 9,29,1 25 | 3,33,1 26 | 44,30,3 27 | 39,44,1 28 | 7,5,2 29 | 49,15,3 30 | 4,50,1 31 | 21,45,1 32 | 34,27,3 33 | 47,45,2 34 | 8,1,3 35 | 31,12,3 36 | 31,10,3 37 | 41,10,3 38 | 27,45,1 39 | 13,50,1 40 | 14,13,2 41 | 26,13,3 42 | 23,32,1 43 | 50,34,3 44 | 11,14,2 45 | 48,28,3 46 | 33,20,3 47 | 3,25,1 48 | 1,27,1 49 | 31,4,3 50 | 3,29,1 -------------------------------------------------------------------------------- /hops/tests/test_resources/mnist/img_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/hops/tests/test_resources/mnist/img_1.jpg -------------------------------------------------------------------------------- /hops/tests/test_resources/online_featurestore_connector.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "featurestoreJdbcConnectorDTO", 3 | "description": "JDBC connection to Hopsworks Project Online Feature Store NDB Database for user: demo_featurestore_admin000_meb1", 4 | "featurestoreId": 107, 5 | "id": 140, 6 | "name": "demo_featurestore_admin000_meb1_onlinefeaturestore", 7 | "storageConnectorType": "JDBC", 8 | "arguments": "password=ccwEdEzjZJVtCmyaIjMKCrVRWwZVbgVA,user=demo_featurestore_admin000_meb1", 9 | "connectionString": "jdbc:mysql://10.0.2.15:3306/demo_featurestore_admin000" 10 | } -------------------------------------------------------------------------------- /hops/tests/test_resources/players_features.csv: -------------------------------------------------------------------------------- 1 | team_id,average_player_rating,average_player_age,average_player_worth,sum_player_rating,sum_player_age,sum_player_worth 2 | 6,1311.2384,24.85,1435.2465,131123.84,2485.0,143524.64 3 | 16,502.2414,25.45,576.87787,50224.14,2545.0,57687.785 4 | 20,372.3458,25.4,417.08722,37234.58,2540.0,41708.723 5 | 40,226.29305,25.91,199.3462,22629.305,2591.0,19934.621 6 | 9,881.2983,25.78,888.29443,88129.83,2578.0,88829.445 7 | 17,467.7938,26.01,490.94702,46779.38,2601.0,49094.703 8 | 23,327.6003,26.63,315.4794,32760.031,2663.0,31547.941 9 | 39,205.15598,26.18,194.33916,20515.598,2618.0,19433.916 10 | 11,657.3932,25.66,674.28235,65739.32,2566.0,67428.234 11 | 33,231.50545,26.34,231.3577,23150.545,2634.0,23135.77 12 | 26,322.69797,25.65,307.87268,32269.797,2565.0,30787.268 13 | 27,297.79196,25.5,298.78235,29779.197,2550.0,29878.234 14 | 15,562.399,25.36,505.629,56239.9,2536.0,50562.9 15 | 43,181.49428,26.18,179.36293,18149.428,2618.0,17936.293 16 | 46,178.7692,25.75,156.81357,17876.92,2575.0,15681.356 17 | 21,389.65762,25.79,405.4787,38965.76,2579.0,40547.87 18 | 36,229.57967,25.67,221.79488,22957.967,2567.0,22179.488 19 | 1,7191.8633,25.88,7920.4326,719186.3,2588.0,792043.25 20 | 3,2814.018,25.22,2823.123,281401.8,2522.0,282312.3 21 | 48,150.96327,25.51,168.30017,15096.327,2551.0,16830.018 22 | 5,1654.0731,27.0,1812.2825,165407.31,2700.0,181228.25 23 | 24,298.03302,25.79,364.08966,29803.3,2579.0,36408.965 24 | 2,4143.072,25.34,4306.55,414307.2,2534.0,430655.0 25 | 4,2027.4081,25.44,2211.348,202740.81,2544.0,221134.8 26 | 7,1480.7965,25.04,1198.3541,148079.66,2504.0,119835.414 27 | 38,216.5106,25.52,182.56189,21651.06,2552.0,18256.19 28 | 22,401.6868,25.11,347.92566,40168.68,2511.0,34792.566 29 | 13,589.41315,24.56,523.0428,58941.316,2456.0,52304.28 30 | 19,401.16718,25.9,427.6141,40116.72,2590.0,42761.41 31 | 8,978.9921,26.25,958.2258,97899.21,2625.0,95822.586 32 | 30,262.44653,24.34,252.60298,26244.654,2434.0,25260.299 33 | 34,240.39302,25.71,223.71338,24039.303,2571.0,22371.338 34 | 44,160.23477,25.9,187.15599,16023.478,2590.0,18715.6 35 | 12,602.10645,24.82,639.6739,60210.645,2482.0,63967.387 36 | 41,218.2798,24.89,199.7587,21827.98,2489.0,19975.87 37 | 49,158.0873,25.42,159.71931,15808.7295,2542.0,15971.932 38 | 32,251.32693,25.26,226.71484,25132.693,2526.0,22671.484 39 | 47,178.67752,25.35,142.29665,17867.752,2535.0,14229.664 40 | 37,227.61397,25.67,184.85991,22761.396,2567.0,18485.99 41 | 35,209.48027,26.03,265.3206,20948.027,2603.0,26532.059 42 | 10,852.43604,24.92,817.88556,85243.6,2492.0,81788.555 43 | 25,348.151,25.3,329.2096,34815.1,2530.0,32920.96 44 | 14,620.2139,25.81,596.4748,62021.39,2581.0,59647.48 45 | 42,197.91428,25.38,203.81004,19791.428,2538.0,20381.004 46 | 18,501.61844,25.01,434.1046,50161.844,2501.0,43410.46 47 | 50,158.26953,25.73,153.84088,15826.953,2573.0,15384.088 48 | 29,269.89728,26.29,267.72226,26989.729,2629.0,26772.225 49 | 31,240.30573,24.63,231.8708,24030.572,2463.0,23187.08 50 | 28,297.92935,25.63,280.11465,29792.936,2563.0,28011.465 51 | 45,180.55344,25.66,185.96408,18055.344,2566.0,18596.408 52 | -------------------------------------------------------------------------------- /hops/tests/test_resources/season_scores_features.csv: -------------------------------------------------------------------------------- 1 | team_id,average_position,sum_position 2 | 26,55.15,1103.0 3 | 27,57.1,1142.0 4 | 9,34.35,687.0 5 | 17,40.3,806.0 6 | 6,28.15,563.0 7 | 16,42.05,841.0 8 | 20,41.05,821.0 9 | 40,63.3,1266.0 10 | 23,55.0,1100.0 11 | 39,61.6,1232.0 12 | 11,32.25,645.0 13 | 33,57.2,1144.0 14 | 15,33.0,660.0 15 | 43,74.15,1483.0 16 | 28,53.6,1072.0 17 | 1,29.05,581.0 18 | 5,29.8,596.0 19 | 42,62.3,1246.0 20 | 8,31.5,630.0 21 | 31,59.4,1188.0 22 | 34,60.65,1213.0 23 | 44,66.2,1324.0 24 | 22,50.45,1009.0 25 | 47,73.95,1479.0 26 | 13,31.95,639.0 27 | 3,28.85,577.0 28 | 48,76.05,1521.0 29 | 19,48.1,962.0 30 | 41,65.55,1311.0 31 | 37,58.45,1169.0 32 | 35,61.8,1236.0 33 | 4,27.1,542.0 34 | 49,80.35,1607.0 35 | 10,40.95,819.0 36 | 50,71.95,1439.0 37 | 45,65.9,1318.0 38 | 38,60.55,1211.0 39 | 29,53.85,1077.0 40 | 21,46.8,936.0 41 | 32,56.4,1128.0 42 | 2,27.1,542.0 43 | 30,54.15,1083.0 44 | 46,73.65,1473.0 45 | 18,39.2,784.0 46 | 36,62.45,1249.0 47 | 12,36.25,725.0 48 | 7,31.25,625.0 49 | 25,53.0,1060.0 50 | 24,45.25,905.0 51 | 14,36.0,720.0 52 | -------------------------------------------------------------------------------- /hops/tests/test_resources/spark-avro_2.11-2.4.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/hops/tests/test_resources/spark-avro_2.11-2.4.0.jar -------------------------------------------------------------------------------- /hops/tests/test_resources/spark-tensorflow-connector_2.11-1.12.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/hops/tests/test_resources/spark-tensorflow-connector_2.11-1.12.0.jar -------------------------------------------------------------------------------- /hops/tests/test_resources/teams_features.csv: -------------------------------------------------------------------------------- 1 | team_budget,team_id,team_position 2 | 12957.076,1,1 3 | 2403.3704,2,2 4 | 3390.3755,3,3 5 | 13547.429,4,4 6 | 9678.333,5,5 7 | 7307.94,6,6 8 | 9469.991,7,7 9 | 2248.776,8,8 10 | 12474.419,9,9 11 | 16107.08,10,10 12 | 4888.2324,11,11 13 | 6101.972,12,12 14 | 21319.533,13,13 15 | 11698.139,14,14 16 | 7683.7227,15,15 17 | 7326.092,16,16 18 | 1621.1936,17,17 19 | 10477.929,18,18 20 | 13022.441,19,19 21 | 3555.235,20,20 22 | 12494.656,21,21 23 | 12433.238,22,22 24 | 10290.323,23,23 25 | 760.8729,24,24 26 | 930.3974,25,25 27 | 16758.066,26,26 28 | 3966.3591,27,27 29 | 3839.0754,28,28 30 | 8516.278,29,29 31 | 6907.2817,30,30 32 | 12514.562,31,31 33 | 4969.735,32,32 34 | 8154.7256,33,33 35 | 1587.0897,34,34 36 | 787.91376,35,35 37 | 14580.948,36,36 38 | 9290.638,37,37 39 | 8086.2744,38,38 40 | 20347.281,39,39 41 | 910.39325,40,40 42 | 1583.5911,41,41 43 | 9775.455,42,42 44 | 4134.0903,43,43 45 | 12510.704,44,44 46 | 12344.707,45,45 47 | 11169.979,46,46 48 | 15072.062,47,47 49 | 11296.577,48,48 50 | 14551.417,49,49 51 | 8320.65,50,50 52 | -------------------------------------------------------------------------------- /hops/tests/test_resources/token.jwt: -------------------------------------------------------------------------------- 1 | eyJraWQiOiI0OSIsInR5cCI6IkpXVCIsImFsZyI6IkhTNTEyIn0.eyJhdWQiOiJqb2IiLCJzdWIiOiJtZWIxMDAwMCIsIm5iZiI6MTU1Mzg1NjAwNywicmVuZXdhYmxlIjpmYWxzZSwiZXhwTGVld2F5IjozMDAsInJvbGVzIjpbIkhPUFNfVVNFUiJdLCJpc3MiOiJob3Bzd29ya3NAbG9naWNhbGNsb2Nrcy5jb20iLCJleHAiOjE1NTM4NTc4MDcsImlhdCI6MTU1Mzg1NjAwNywianRpIjoiODNmMWQwOTgtM2Y0Mi00NDQ2LThkODctNGNhMjkxNWM2MGVhIn0.y5eFyGI8ZKow1GAV7jdFA_wm6aK14dZsYrLUTNIWp4-6eYpdlyIrSERyzlw2WZkr1-Ha0MIOgux2lR76uPK97g -------------------------------------------------------------------------------- /hops/tls.py: -------------------------------------------------------------------------------- 1 | """ 2 | A module for getting TLS certificates in YARN containers, used for setting up Kafka inside Jobs/Notebooks on Hops. 3 | """ 4 | 5 | import string 6 | import base64 7 | import textwrap 8 | from hops import constants 9 | import os 10 | from pathlib import Path 11 | import tempfile 12 | 13 | try: 14 | import jks 15 | except: 16 | pass 17 | 18 | 19 | def _get_key_store_path(): 20 | """ 21 | Get keystore path 22 | 23 | Returns: 24 | keystore path 25 | """ 26 | k_certificate = Path(constants.SSL_CONFIG.K_CERTIFICATE_CONFIG) 27 | if k_certificate.exists(): 28 | return k_certificate 29 | else: 30 | username = os.environ['HADOOP_USER_NAME'] 31 | material_directory = Path(os.environ['MATERIAL_DIRECTORY']) 32 | return material_directory.joinpath(username + constants.SSL_CONFIG.KEYSTORE_SUFFIX) 33 | 34 | 35 | def get_key_store(): 36 | return str(_get_key_store_path()) 37 | 38 | 39 | def _get_trust_store_path(): 40 | """ 41 | Get truststore location 42 | 43 | Returns: 44 | truststore location 45 | """ 46 | t_certificate = Path(constants.SSL_CONFIG.T_CERTIFICATE_CONFIG) 47 | if t_certificate.exists(): 48 | return str(t_certificate) 49 | else: 50 | username = os.environ['HADOOP_USER_NAME'] 51 | material_directory = Path(os.environ['MATERIAL_DIRECTORY']) 52 | return str(material_directory.joinpath(username + constants.SSL_CONFIG.TRUSTSTORE_SUFFIX)) 53 | 54 | 55 | def get_trust_store(): 56 | return str(_get_trust_store_path()) 57 | 58 | 59 | def _get_cert_pw(): 60 | """ 61 | Get keystore password from local container 62 | 63 | Returns: 64 | Certificate password 65 | """ 66 | pwd_path = Path(constants.SSL_CONFIG.CRYPTO_MATERIAL_PASSWORD) 67 | if not pwd_path.exists(): 68 | username = os.environ['HADOOP_USER_NAME'] 69 | material_directory = Path(os.environ['MATERIAL_DIRECTORY']) 70 | pwd_path = material_directory.joinpath(username + constants.SSL_CONFIG.PASSWORD_SUFFIX) 71 | 72 | with pwd_path.open() as f: 73 | return f.read() 74 | 75 | 76 | def get_key_store_cert(): 77 | """ 78 | Get keystore certificate from local container 79 | 80 | Returns: 81 | Certificate password 82 | """ 83 | cert_path = _get_key_store_path() 84 | 85 | if not cert_path.exists(): 86 | raise AssertionError('k_certificate is not present in directory: {}'.format(str(cert_path))) 87 | 88 | # read as bytes, don't try to use utf-8 encoding 89 | with cert_path.open("rb") as f: 90 | key_store_cert = f.read() 91 | key_store_cert = base64.b64encode(key_store_cert) 92 | 93 | return key_store_cert 94 | 95 | 96 | def get_key_store_pwd(): 97 | """ 98 | Get keystore password 99 | 100 | Returns: 101 | keystore password 102 | """ 103 | return _get_cert_pw() 104 | 105 | 106 | def get_trust_store_pwd(): 107 | """ 108 | Get truststore password 109 | 110 | Returns: 111 | truststore password 112 | """ 113 | return _get_cert_pw() 114 | 115 | 116 | def _bytes_to_pem_str(der_bytes, pem_type): 117 | """ 118 | Utility function for creating PEM files 119 | 120 | Args: 121 | der_bytes: DER encoded bytes 122 | pem_type: type of PEM, e.g Certificate, Private key, or RSA private key 123 | 124 | Returns: 125 | PEM String for a DER-encoded certificate or private key 126 | """ 127 | pem_str = "" 128 | pem_str = pem_str + "-----BEGIN {}-----".format(pem_type) + "\n" 129 | pem_str = pem_str + "\r\n".join(textwrap.wrap(base64.b64encode(der_bytes).decode('ascii'), 64)) + "\n" 130 | pem_str = pem_str + "-----END {}-----".format(pem_type) + "\n" 131 | return pem_str 132 | 133 | 134 | def _convert_jks_to_pem(jks_path, keystore_pw): 135 | """ 136 | Converts a keystore JKS that contains client private key, 137 | client certificate and CA certificate that was used to 138 | sign the certificate, to three PEM-format strings. 139 | 140 | Args: 141 | :jks_path: path to the JKS file 142 | :pw: password for decrypting the JKS file 143 | 144 | Returns: 145 | strings: (client_cert, client_key, ca_cert) 146 | """ 147 | # load the keystore and decrypt it with password 148 | ks = jks.KeyStore.load(jks_path, keystore_pw, try_decrypt_keys=True) 149 | private_keys_certs = "" 150 | private_keys = "" 151 | ca_certs = "" 152 | 153 | # Convert private keys and their certificates into PEM format and append to string 154 | for alias, pk in ks.private_keys.items(): 155 | if pk.algorithm_oid == jks.util.RSA_ENCRYPTION_OID: 156 | private_keys = private_keys + _bytes_to_pem_str(pk.pkey, "RSA PRIVATE KEY") 157 | else: 158 | private_keys = private_keys + _bytes_to_pem_str(pk.pkey_pkcs8, "PRIVATE KEY") 159 | for c in pk.cert_chain: 160 | # c[0] contains type of cert, i.e X.509 161 | private_keys_certs = private_keys_certs + _bytes_to_pem_str(c[1], "CERTIFICATE") 162 | 163 | # Convert CA Certificates into PEM format and append to string 164 | for alias, c in ks.certs.items(): 165 | ca_certs = ca_certs + _bytes_to_pem_str(c.cert, "CERTIFICATE") 166 | return private_keys_certs, private_keys, ca_certs 167 | 168 | 169 | def _write_pem(jks_key_store_path, jks_trust_store_path, keystore_pw, client_key_cert_path, client_key_path, 170 | ca_cert_path): 171 | """ 172 | Converts the JKS keystore, JKS truststore, and the root ca.pem 173 | client certificate, client key, and ca certificate 174 | 175 | Args: 176 | :jks_key_store_path: path to the JKS keystore 177 | :jks_trust_store_path: path to the JKS truststore 178 | :keystore_pw: path to file with passphrase for the keystores 179 | :client_key_cert_path: path to write the client's certificate for its private key in PEM format 180 | :client_key_path: path to write the client's private key in PEM format 181 | :ca_cert_path: path to write the chain of CA certificates required to validate certificates 182 | """ 183 | keystore_key_cert, keystore_key, keystore_ca_cert = _convert_jks_to_pem(jks_key_store_path, keystore_pw) 184 | truststore_key_cert, truststore_key, truststore_ca_cert = _convert_jks_to_pem(jks_trust_store_path, keystore_pw) 185 | with client_key_cert_path.open("w") as f: 186 | f.write(keystore_key_cert) 187 | with client_key_path.open("w") as f: 188 | f.write(keystore_key) 189 | with ca_cert_path.open("w") as f: 190 | f.write(keystore_ca_cert + truststore_ca_cert) 191 | 192 | 193 | def get_client_certificate_location(): 194 | """ 195 | Get location of client certificate (PEM format) for the private key signed by trusted CA 196 | used for 2-way TLS authentication, for example with Kafka cluster 197 | 198 | Returns: 199 | string path to client certificate in PEM format 200 | """ 201 | certificate_path = pems_dir(constants.SSL_CONFIG.PEM_CLIENT_CERTIFICATE_CONFIG) 202 | if not certificate_path.exists(): 203 | _write_pems() 204 | return str(certificate_path) 205 | 206 | 207 | def get_client_key_location(): 208 | """ 209 | Get location of client private key (PEM format) 210 | used for for 2-way TLS authentication, for example with Kafka cluster 211 | 212 | Returns: 213 | string path to client private key in PEM format 214 | """ 215 | # Convert JKS to PEMs if they don't exists already 216 | key_path = pems_dir(constants.SSL_CONFIG.PEM_CLIENT_KEY_CONFIG) 217 | if not key_path.exists(): 218 | _write_pems() 219 | return str(key_path) 220 | 221 | 222 | def get_ca_chain_location(): 223 | """ 224 | Get location of chain of CA certificates (PEM format) that are required to validate the 225 | private key certificate of the client 226 | used for 2-way TLS authentication, for example with Kafka cluster 227 | 228 | Returns: 229 | string path to ca chain of certificate 230 | """ 231 | ca_chain_path = pems_dir(constants.SSL_CONFIG.PEM_CA_CHAIN_CERTIFICATE_CONFIG) 232 | if not ca_chain_path.exists(): 233 | _write_pems() 234 | return str(ca_chain_path) 235 | 236 | 237 | def pems_dir(pem): 238 | """ 239 | Get location of a pem file 240 | 241 | Returns: 242 | path to a pem file 243 | """ 244 | pem_dir = Path(tempfile.gettempdir()) 245 | return pem_dir.joinpath(pem) 246 | 247 | 248 | def _write_pems(): 249 | """ 250 | Converts JKS keystore file into PEM to be compatible with Python libraries 251 | """ 252 | t_jks_path = get_trust_store() 253 | k_jks_path = get_key_store() 254 | client_certificate_path = pems_dir(constants.SSL_CONFIG.PEM_CLIENT_CERTIFICATE_CONFIG) 255 | client_key_path = pems_dir(constants.SSL_CONFIG.PEM_CLIENT_KEY_CONFIG) 256 | ca_chain_path = pems_dir(constants.SSL_CONFIG.PEM_CA_CHAIN_CERTIFICATE_CONFIG) 257 | 258 | _write_pem(k_jks_path, t_jks_path, get_key_store_pwd(), client_certificate_path, client_key_path, ca_chain_path) 259 | -------------------------------------------------------------------------------- /hops/user.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Utility functions to manage users in Hopsworks. 4 | 5 | """ 6 | import json 7 | from hops import constants, util 8 | 9 | import logging 10 | log = logging.getLogger(__name__) 11 | 12 | def create_user(new_user): 13 | """ 14 | Create a user in Hopsworks. Registers and activates a user with role HOPS_USER. 15 | 16 | Example usage: 17 | 18 | >>> from hops import util, user 19 | >>> new_user = {"firstName":"Joe","lastName":"Doe","email":"joe@hopsworks.ai","telephoneNum":"", 20 | >>> "chosenPassword":"Admin123","repeatedPassword":"Admin123", 21 | >>> "securityQuestion":"What is your oldest sibling's middle name?","securityAnswer":"Admin123", 22 | >>> "tos":"true","authType":"Mobile","twoFactor":"false","toursEnabled":"true","orgName":"","dep":"", 23 | >>> "street":"","city":"","postCode":"","country":"","testUser":"false"} 24 | >>> util.connect("localhost", api_key="api_key_file") 25 | >>> user.create(new_user) 26 | 27 | Args: 28 | :new_user: Dict with the new user attributes 29 | 30 | Returns: 31 | None 32 | """ 33 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 34 | # Register user 35 | util.http(constants.DELIMITERS.SLASH_DELIMITER + \ 36 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 37 | constants.REST_CONFIG.HOPSWORKS_AUTH_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 38 | constants.REST_CONFIG.HOPSWORKS_AUTH_RESOURCE_REGISTER, 39 | headers=headers, 40 | method=constants.HTTP_CONFIG.HTTP_POST, 41 | data=json.dumps(new_user)) 42 | 43 | # Get user id 44 | response = util.http(constants.DELIMITERS.SLASH_DELIMITER + \ 45 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 46 | constants.REST_CONFIG.HOPSWORKS_ADMIN_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 47 | constants.REST_CONFIG.HOPSWORKS_USERS_RESOURCE + "?filter_by=user_email:" + new_user["email"], 48 | headers=headers, 49 | method=constants.HTTP_CONFIG.HTTP_GET) 50 | user_profile = response['items'][0] 51 | user_profile["status"] = "VERIFIED_ACCOUNT" 52 | # Verify user 53 | util.http(constants.DELIMITERS.SLASH_DELIMITER + \ 54 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 55 | constants.REST_CONFIG.HOPSWORKS_ADMIN_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 56 | constants.REST_CONFIG.HOPSWORKS_USERS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 57 | str(user_profile['id']) + constants.DELIMITERS.SLASH_DELIMITER, 58 | headers=headers, 59 | method=constants.HTTP_CONFIG.HTTP_PUT, 60 | data=json.dumps(user_profile)) 61 | # Accept user 62 | response = util.http(constants.DELIMITERS.SLASH_DELIMITER + \ 63 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 64 | constants.REST_CONFIG.HOPSWORKS_ADMIN_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 65 | constants.REST_CONFIG.HOPSWORKS_USERS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 66 | str(user_profile['id']) + constants.DELIMITERS.SLASH_DELIMITER + "accepted", 67 | headers=headers, 68 | method=constants.HTTP_CONFIG.HTTP_PUT, 69 | data=json.dumps(user_profile)) 70 | log.info(response) 71 | -------------------------------------------------------------------------------- /hops/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '4.0.0.0.rc1' 2 | -------------------------------------------------------------------------------- /hops/xattr.py: -------------------------------------------------------------------------------- 1 | """ 2 | API for attaching, detaching, and reading extended metadata to HopsFS files/directories. 3 | It uses the Hopsworks /xattrs REST API 4 | """ 5 | from hops import constants, util, hdfs 6 | from hops.exceptions import RestAPIError 7 | import urllib 8 | 9 | 10 | def set_xattr(hdfs_path, xattr_name, value): 11 | """ 12 | Attach an extended attribute to an hdfs_path 13 | 14 | Args: 15 | :hdfs_path: path of a file or directory 16 | :xattr_name: name of the extended attribute 17 | :value: value of the extended attribute 18 | 19 | Returns: 20 | None 21 | """ 22 | value = str(value) 23 | hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path)) 24 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 25 | method = constants.HTTP_CONFIG.HTTP_PUT 26 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 27 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 28 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 29 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 30 | constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 31 | hdfs_path + constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \ 32 | constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name 33 | response = util.send_request(method, resource_url, data=value, headers=headers) 34 | response_object = response.json() 35 | if response.status_code >= 400: 36 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 37 | raise RestAPIError("Could not attach extened attributes from a path (url: {}), server response: \n " \ 38 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 39 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 40 | 41 | 42 | def get_xattr(hdfs_path, xattr_name=None): 43 | """ 44 | Get the extended attribute attached to an hdfs_path. 45 | 46 | Args: 47 | :hdfs_path: path of a file or directory 48 | :xattr_name: name of the extended attribute 49 | 50 | Returns: 51 | A dictionary with the extended attribute(s) as key value pair(s). If the :xattr_name is None, 52 | the API returns all associated extended attributes. 53 | """ 54 | hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path)) 55 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 56 | method = constants.HTTP_CONFIG.HTTP_GET 57 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 58 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 59 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 60 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 61 | constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 62 | hdfs_path 63 | if xattr_name is not None: 64 | resource_url += constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \ 65 | constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name 66 | 67 | response = util.send_request(method, resource_url, headers=headers) 68 | response_object = response.json() 69 | if response.status_code >= 400: 70 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 71 | raise RestAPIError("Could not get extened attributes attached to a path (url: {}), server response: \n " \ 72 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 73 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) 74 | 75 | results = {} 76 | for item in response_object["items"]: 77 | results[item["name"]] = item["value"] 78 | return results 79 | 80 | 81 | def remove_xattr(hdfs_path, xattr_name): 82 | """ 83 | Remove an extended attribute attached to an hdfs_path 84 | 85 | Args: 86 | :hdfs_path: path of a file or directory 87 | :xattr_name: name of the extended attribute 88 | 89 | Returns: 90 | None 91 | """ 92 | hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path)) 93 | headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} 94 | method = constants.HTTP_CONFIG.HTTP_DELETE 95 | resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ 96 | constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 97 | constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 98 | hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ 99 | constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ 100 | hdfs_path + constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \ 101 | constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name 102 | response = util.send_request(method, resource_url, headers=headers) 103 | if response.status_code >= 400: 104 | response_object = response.json() 105 | error_code, error_msg, user_msg = util._parse_rest_error(response_object) 106 | raise RestAPIError("Could not remove extened attributes from a path (url: {}), server response: \n " \ 107 | "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( 108 | resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) -------------------------------------------------------------------------------- /imgs/add-python-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/add-python-module.png -------------------------------------------------------------------------------- /imgs/driver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/driver.png -------------------------------------------------------------------------------- /imgs/executor-hdfs-log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/executor-hdfs-log.png -------------------------------------------------------------------------------- /imgs/executor-stderr1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/executor-stderr1.png -------------------------------------------------------------------------------- /imgs/executor-stderr2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/executor-stderr2.png -------------------------------------------------------------------------------- /imgs/executor-stderr3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/executor-stderr3.png -------------------------------------------------------------------------------- /imgs/executor-stderr4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/executor-stderr4.png -------------------------------------------------------------------------------- /imgs/feature_plots1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/feature_plots1.png -------------------------------------------------------------------------------- /imgs/feature_plots2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/feature_plots2.png -------------------------------------------------------------------------------- /imgs/hopsml-pyspark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logicalclocks/hops-util-py/c78181e2a6fa06197afad7c8666389d045b65d77/imgs/hopsml-pyspark.png -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore 4 | 5 | log_cli = 1 6 | log_cli_level = INFO 7 | log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) 8 | log_cli_date_format=%Y-%m-%d %H:%M:%S -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup, find_packages 3 | 4 | exec(open('hops/version.py').read()) 5 | 6 | def read(fname): 7 | try: 8 | return open(os.path.join(os.path.dirname(__file__), fname), encoding='utf8').read() #python3 9 | except: 10 | return open(os.path.join(os.path.dirname(__file__), fname)).read() #python2 11 | 12 | setup( 13 | name='hops', 14 | version=__version__, 15 | install_requires=[ 16 | 'numpy', 17 | 'pandas', 18 | 'pyjks', 19 | 'pyhopshive[thrift]', 20 | 'boto3', 21 | 'pyopenssl', 22 | 'idna', 23 | 'cryptography', 24 | 'dnspython==2.6.1', 25 | 'nvidia-ml-py3==7.352.0', 26 | 'requests' 27 | ], 28 | extras_require={ 29 | 'pydoop': ['pydoop'], 30 | 'tf': ['tensorflow'], 31 | 'docs': [ 32 | 'sphinx', 33 | 'sphinx-autobuild', 34 | 'recommonmark', 35 | 'sphinx_rtd_theme', 36 | 'jupyter_sphinx_theme' 37 | ], 38 | 'test': [ 39 | 'mock', 40 | 'pytest', 41 | ], 42 | 'spark': ['pyspark==2.4.3'], 43 | 'plotting': ['matplotlib', 'seaborn'] 44 | }, 45 | author='Robin Andersson', 46 | author_email='robin.eric.andersson@gmail.com', 47 | description='Client library for interacting with Hopsworks, a full-stack platform for scale-out data science.', 48 | license='Apache License 2.0', 49 | keywords='Hops, Hadoop, TensorFlow, Spark', 50 | url='https://github.com/logicalclocks/hops-util-py', 51 | download_url='http://snurran.sics.se/hops/hops-util-py/hops-' + __version__ + '.tar.gz', 52 | packages=find_packages(exclude=['tests']), 53 | long_description=read('README.rst'), 54 | classifiers=[ 55 | 'Development Status :: 5 - Production/Stable', 56 | 'Topic :: Utilities', 57 | 'License :: OSI Approved :: Apache Software License', 58 | 'Programming Language :: Python :: 3', 59 | ] 60 | ) 61 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.common.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.common package 2 | ========================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.common.featurestore\_entity module 8 | -------------------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.dao.common.featurestore_entity 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.featurestore\_impl.dao.common.featurestore\_metadata module 16 | ---------------------------------------------------------------- 17 | 18 | .. automodule:: hops.featurestore_impl.dao.common.featurestore_metadata 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: hops.featurestore_impl.dao.common 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.datasets.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.datasets package 2 | ============================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.datasets.external\_training\_dataset module 8 | ----------------------------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.dao.datasets.external_training_dataset 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.featurestore\_impl.dao.datasets.hopsfs\_training\_dataset module 16 | --------------------------------------------------------------------- 17 | 18 | .. automodule:: hops.featurestore_impl.dao.datasets.hopsfs_training_dataset 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.featurestore\_impl.dao.datasets.training\_dataset module 24 | ------------------------------------------------------------- 25 | 26 | .. automodule:: hops.featurestore_impl.dao.datasets.training_dataset 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: hops.featurestore_impl.dao.datasets 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.featuregroups.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.featuregroups package 2 | ================================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.featuregroups.cached\_featuregroup module 8 | --------------------------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.dao.featuregroups.cached_featuregroup 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.featurestore\_impl.dao.featuregroups.featuregroup module 16 | ------------------------------------------------------------- 17 | 18 | .. automodule:: hops.featurestore_impl.dao.featuregroups.featuregroup 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.featurestore\_impl.dao.featuregroups.on\_demand\_featuregroup module 24 | ------------------------------------------------------------------------- 25 | 26 | .. automodule:: hops.featurestore_impl.dao.featuregroups.on_demand_featuregroup 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hops.featurestore\_impl.dao.featuregroups.online\_featuregroup module 32 | --------------------------------------------------------------------- 33 | 34 | .. automodule:: hops.featurestore_impl.dao.featuregroups.online_featuregroup 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: hops.featurestore_impl.dao.featuregroups 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.features.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.features package 2 | ============================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.features.feature module 8 | --------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.dao.features.feature 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.dao.features 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.featurestore.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.featurestore package 2 | ================================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.featurestore.featurestore module 8 | ------------------------------------------------------------ 9 | 10 | .. automodule:: hops.featurestore_impl.dao.featurestore.featurestore 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.dao.featurestore 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao package 2 | =================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | hops.featurestore_impl.dao.common 10 | hops.featurestore_impl.dao.datasets 11 | hops.featurestore_impl.dao.featuregroups 12 | hops.featurestore_impl.dao.features 13 | hops.featurestore_impl.dao.featurestore 14 | hops.featurestore_impl.dao.settings 15 | hops.featurestore_impl.dao.stats 16 | hops.featurestore_impl.dao.storageconnectors 17 | 18 | Module contents 19 | --------------- 20 | 21 | .. automodule:: hops.featurestore_impl.dao 22 | :members: 23 | :undoc-members: 24 | :show-inheritance: 25 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.settings.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.settings package 2 | ============================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.settings.featurestore\_settings module 8 | ------------------------------------------------------------------ 9 | 10 | .. automodule:: hops.featurestore_impl.dao.settings.featurestore_settings 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.dao.settings 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.stats.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.stats package 2 | ========================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.stats.cluster module 8 | ------------------------------------------------ 9 | 10 | .. automodule:: hops.featurestore_impl.dao.stats.cluster 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.featurestore\_impl.dao.stats.cluster\_analysis module 16 | ---------------------------------------------------------- 17 | 18 | .. automodule:: hops.featurestore_impl.dao.stats.cluster_analysis 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.featurestore\_impl.dao.stats.correlation\_matrix module 24 | ------------------------------------------------------------ 25 | 26 | .. automodule:: hops.featurestore_impl.dao.stats.correlation_matrix 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hops.featurestore\_impl.dao.stats.correlation\_value module 32 | ----------------------------------------------------------- 33 | 34 | .. automodule:: hops.featurestore_impl.dao.stats.correlation_value 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hops.featurestore\_impl.dao.stats.datapoint module 40 | -------------------------------------------------- 41 | 42 | .. automodule:: hops.featurestore_impl.dao.stats.datapoint 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | hops.featurestore\_impl.dao.stats.descriptive\_stats module 48 | ----------------------------------------------------------- 49 | 50 | .. automodule:: hops.featurestore_impl.dao.stats.descriptive_stats 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | hops.featurestore\_impl.dao.stats.descriptive\_stats\_metric\_value module 56 | -------------------------------------------------------------------------- 57 | 58 | .. automodule:: hops.featurestore_impl.dao.stats.descriptive_stats_metric_value 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | hops.featurestore\_impl.dao.stats.descriptive\_stats\_metric\_values module 64 | --------------------------------------------------------------------------- 65 | 66 | .. automodule:: hops.featurestore_impl.dao.stats.descriptive_stats_metric_values 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | hops.featurestore\_impl.dao.stats.feature\_correlation module 72 | ------------------------------------------------------------- 73 | 74 | .. automodule:: hops.featurestore_impl.dao.stats.feature_correlation 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | hops.featurestore\_impl.dao.stats.feature\_histogram module 80 | ----------------------------------------------------------- 81 | 82 | .. automodule:: hops.featurestore_impl.dao.stats.feature_histogram 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | hops.featurestore\_impl.dao.stats.feature\_histograms module 88 | ------------------------------------------------------------ 89 | 90 | .. automodule:: hops.featurestore_impl.dao.stats.feature_histograms 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | hops.featurestore\_impl.dao.stats.histogram\_bin module 96 | ------------------------------------------------------- 97 | 98 | .. automodule:: hops.featurestore_impl.dao.stats.histogram_bin 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | hops.featurestore\_impl.dao.stats.statistics module 104 | --------------------------------------------------- 105 | 106 | .. automodule:: hops.featurestore_impl.dao.stats.statistics 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | 112 | Module contents 113 | --------------- 114 | 115 | .. automodule:: hops.featurestore_impl.dao.stats 116 | :members: 117 | :undoc-members: 118 | :show-inheritance: 119 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.dao.storageconnectors.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.dao.storageconnectors package 2 | ===================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.dao.storageconnectors.hopsfs\_connector module 8 | ---------------------------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.dao.storageconnectors.hopsfs_connector 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.featurestore\_impl.dao.storageconnectors.jdbc\_connector module 16 | -------------------------------------------------------------------- 17 | 18 | .. automodule:: hops.featurestore_impl.dao.storageconnectors.jdbc_connector 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.featurestore\_impl.dao.storageconnectors.s3\_connector module 24 | ------------------------------------------------------------------ 25 | 26 | .. automodule:: hops.featurestore_impl.dao.storageconnectors.s3_connector 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: hops.featurestore_impl.dao.storageconnectors 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.exceptions.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.exceptions package 2 | ========================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.exceptions.exceptions module 8 | ---------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.exceptions.exceptions 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.exceptions 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.featureframes.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.featureframes package 2 | ============================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.featureframes.FeatureFrame module 8 | --------------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.featureframes.FeatureFrame 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.featureframes 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.online_featurestore.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.online\_featurestore package 2 | ==================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.online\_featurestore.online\_featurestore module 8 | ------------------------------------------------------------------------ 9 | 10 | .. automodule:: hops.featurestore_impl.online_featurestore.online_featurestore 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.online_featurestore 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.query_planner.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.query\_planner package 2 | ============================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.query\_planner.f\_query module 8 | ------------------------------------------------------ 9 | 10 | .. automodule:: hops.featurestore_impl.query_planner.f_query 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hops.featurestore\_impl.query\_planner.fg\_query module 16 | ------------------------------------------------------- 17 | 18 | .. automodule:: hops.featurestore_impl.query_planner.fg_query 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.featurestore\_impl.query\_planner.fs\_query module 24 | ------------------------------------------------------- 25 | 26 | .. automodule:: hops.featurestore_impl.query_planner.fs_query 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hops.featurestore\_impl.query\_planner.logical\_query\_plan module 32 | ------------------------------------------------------------------ 33 | 34 | .. automodule:: hops.featurestore_impl.query_planner.logical_query_plan 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hops.featurestore\_impl.query\_planner.query\_planner module 40 | ------------------------------------------------------------ 41 | 42 | .. automodule:: hops.featurestore_impl.query_planner.query_planner 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: hops.featurestore_impl.query_planner 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.rest.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.rest package 2 | ==================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.rest.rest\_rpc module 8 | --------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.rest.rest_rpc 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.rest 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl package 2 | =============================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | hops.featurestore_impl.dao 10 | hops.featurestore_impl.exceptions 11 | hops.featurestore_impl.featureframes 12 | hops.featurestore_impl.online_featurestore 13 | hops.featurestore_impl.query_planner 14 | hops.featurestore_impl.rest 15 | hops.featurestore_impl.util 16 | hops.featurestore_impl.visualizations 17 | 18 | Submodules 19 | ---------- 20 | 21 | hops.featurestore\_impl.core module 22 | ----------------------------------- 23 | 24 | .. automodule:: hops.featurestore_impl.core 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | 29 | 30 | Module contents 31 | --------------- 32 | 33 | .. automodule:: hops.featurestore_impl 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.util.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.util package 2 | ==================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.util.fs\_utils module 8 | --------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.util.fs_utils 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.util 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.featurestore_impl.visualizations.rst: -------------------------------------------------------------------------------- 1 | hops.featurestore\_impl.visualizations package 2 | ============================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hops.featurestore\_impl.visualizations.statistics\_plots module 8 | --------------------------------------------------------------- 9 | 10 | .. automodule:: hops.featurestore_impl.visualizations.statistics_plots 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: hops.featurestore_impl.visualizations 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /source/hops.rst: -------------------------------------------------------------------------------- 1 | hops package 2 | ============ 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | hops.experiment_impl 10 | hops.featurestore_impl 11 | 12 | Submodules 13 | ---------- 14 | 15 | hops.constants module 16 | --------------------- 17 | 18 | .. automodule:: hops.constants 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hops.devices module 24 | ------------------- 25 | 26 | .. automodule:: hops.devices 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hops.exceptions module 32 | ---------------------- 33 | 34 | .. automodule:: hops.exceptions 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hops.experiment module 40 | ---------------------- 41 | 42 | .. automodule:: hops.experiment 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | hops.featurestore module 48 | ------------------------ 49 | 50 | .. automodule:: hops.featurestore 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | hops.hdfs module 56 | ---------------- 57 | 58 | .. automodule:: hops.hdfs 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | hops.hive module 64 | ---------------- 65 | 66 | .. automodule:: hops.hive 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | hops.jobs module 72 | ---------------- 73 | 74 | .. automodule:: hops.jobs 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | hops.kafka module 80 | ----------------- 81 | 82 | .. automodule:: hops.kafka 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | hops.numpy\_helper module 88 | ------------------------- 89 | 90 | .. automodule:: hops.numpy_helper 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | hops.pandas\_helper module 96 | -------------------------- 97 | 98 | .. automodule:: hops.pandas_helper 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | hops.serving module 104 | ------------------- 105 | 106 | .. automodule:: hops.serving 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | hops.tensorboard module 112 | ----------------------- 113 | 114 | .. automodule:: hops.tensorboard 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | hops.tls module 120 | --------------- 121 | 122 | .. automodule:: hops.tls 123 | :members: 124 | :undoc-members: 125 | :show-inheritance: 126 | 127 | hops.util module 128 | ---------------- 129 | 130 | .. automodule:: hops.util 131 | :members: 132 | :undoc-members: 133 | :show-inheritance: 134 | 135 | hops.version module 136 | ------------------- 137 | 138 | .. automodule:: hops.version 139 | :members: 140 | :undoc-members: 141 | :show-inheritance: 142 | 143 | 144 | Module contents 145 | --------------- 146 | 147 | .. automodule:: hops 148 | :members: 149 | :undoc-members: 150 | :show-inheritance: 151 | -------------------------------------------------------------------------------- /source/modules.rst: -------------------------------------------------------------------------------- 1 | hops 2 | ==== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | hops 8 | setup 9 | -------------------------------------------------------------------------------- /source/setup.rst: -------------------------------------------------------------------------------- 1 | setup module 2 | ============ 3 | 4 | .. automodule:: setup 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | --------------------------------------------------------------------------------