├── .gitignore ├── .pypirc ├── .travis.yml ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── base-images-tag.json ├── bin ├── ci-basic.sh ├── ci-product.sh └── install-docker.sh ├── docs ├── Makefile ├── conf.py ├── contributing.rst ├── emr.rst ├── index.rst ├── installation │ ├── advanced-installation-options.rst │ ├── java-installation.rst │ ├── presto-admin-configuration.rst │ ├── presto-admin-installation.rst │ ├── presto-admin-upgrade.rst │ ├── presto-catalog-installation.rst │ ├── presto-cli-installation.rst │ ├── presto-configuration.rst │ ├── presto-port-configuration.rst │ ├── presto-server-installation.rst │ └── troubleshooting-installation.rst ├── presto-admin-cli-options.rst ├── presto-admin-commands.rst ├── quick-start-guide.rst ├── release.rst ├── release │ ├── release-0.1.0.rst │ ├── release-1.1.rst │ ├── release-1.2.rst │ ├── release-1.3.rst │ ├── release-1.4.rst │ ├── release-1.5.rst │ ├── release-2.0.rst │ ├── release-2.1.rst │ └── release-2.2.rst ├── software-requirements.rst ├── ssh-configuration.rst └── user-guide.rst ├── packaging ├── __init__.py ├── bdist_prestoadmin.py └── install-prestoadmin.template ├── prestoadmin ├── __init__.py ├── _version.py ├── catalog.py ├── collect.py ├── config.py ├── configure_cmds.py ├── coordinator.py ├── deploy.py ├── fabric_patches.py ├── file.py ├── main.py ├── mode.py ├── node.py ├── package.py ├── plugin.py ├── presto-admin-logging.ini ├── presto_conf.py ├── prestoclient.py ├── server.py ├── standalone │ ├── __init__.py │ └── config.py ├── topology.py ├── util │ ├── __init__.py │ ├── all_write_handler.py │ ├── application.py │ ├── base_config.py │ ├── constants.py │ ├── exception.py │ ├── fabric_application.py │ ├── fabricapi.py │ ├── filesystem.py │ ├── hiddenoptgroup.py │ ├── httpscacertconnection.py │ ├── local_config_util.py │ ├── parser.py │ ├── presto_config.py │ ├── remote_config_util.py │ ├── validators.py │ └── version_util.py ├── workers.py └── yarn_slider │ ├── __init__.py │ ├── config.py │ ├── server.py │ └── slider.py ├── release.py ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── bare_image_provider.py ├── base_cluster.py ├── base_installer.py ├── base_test_case.py ├── configurable_cluster.py ├── docker_cluster.py ├── integration │ ├── __init__.py │ └── util │ │ ├── __init__.py │ │ ├── data │ │ └── presto-admin-logging.ini │ │ └── test_application.py ├── no_hadoop_bare_image_provider.py ├── product │ ├── __init__.py │ ├── base_product_case.py │ ├── cluster_types.py │ ├── config_dir_utils.py │ ├── constants.py │ ├── image_builder.py │ ├── mode_installers.py │ ├── prestoadmin_installer.py │ ├── resources │ │ ├── configuration_show_config.txt │ │ ├── configuration_show_default.txt │ │ ├── configuration_show_default_master_slave1.txt │ │ ├── configuration_show_default_slave2_slave3.txt │ │ ├── configuration_show_down_node.txt │ │ ├── configuration_show_jvm.txt │ │ ├── configuration_show_log.txt │ │ ├── configuration_show_log_none.txt │ │ ├── configuration_show_node.txt │ │ ├── configuration_show_none.txt │ │ ├── dummy-rpm.rpm │ │ ├── install-admin.sh │ │ ├── install_twice.txt │ │ ├── invalid_json.json │ │ ├── non_root_sudo_warning_text.txt │ │ ├── non_sudo_uninstall.txt │ │ ├── parallel_password_failure.txt │ │ ├── slider-assembly-0.80.0-incubating-all.tar.gz │ │ └── uninstall_twice.txt │ ├── standalone │ │ ├── __init__.py │ │ ├── presto_installer.py │ │ └── test_installation.py │ ├── test_authentication.py │ ├── test_catalog.py │ ├── test_collect.py │ ├── test_configuration.py │ ├── test_control.py │ ├── test_error_handling.py │ ├── test_file.py │ ├── test_installer.py │ ├── test_package_install.py │ ├── test_plugin.py │ ├── test_server_install.py │ ├── test_server_uninstall.py │ ├── test_server_upgrade.py │ ├── test_status.py │ ├── test_topology.py │ ├── timing_test_decorator.py │ └── topology_installer.py ├── rpm │ ├── __init__.py │ └── test_rpm.py └── unit │ ├── __init__.py │ ├── base_unit_case.py │ ├── resources │ ├── empty.txt │ ├── invalid.properties │ ├── invalid_json_conf.json │ ├── server_status_out.txt │ ├── slider-extended-help.txt │ ├── slider-help.txt │ ├── standalone-extended-help.txt │ ├── standalone-help.txt │ ├── valid.config │ ├── valid.properties │ ├── valid_rest_response_level1.txt │ └── valid_rest_response_level2.txt │ ├── standalone │ ├── __init__.py │ └── test_help.py │ ├── test_base_test_case.py │ ├── test_bdist_prestoadmin.py │ ├── test_catalog.py │ ├── test_collect.py │ ├── test_config.py │ ├── test_configure_cmds.py │ ├── test_coordinator.py │ ├── test_deploy.py │ ├── test_expand.py │ ├── test_fabric_patches.py │ ├── test_file.py │ ├── test_main.py │ ├── test_package.py │ ├── test_plugin.py │ ├── test_presto_conf.py │ ├── test_presto_config.py │ ├── test_prestoclient.py │ ├── test_server.py │ ├── test_topology.py │ ├── test_workers.py │ ├── util │ ├── __init__.py │ ├── test_application.py │ ├── test_base_config.py │ ├── test_exception.py │ ├── test_fabric_application.py │ ├── test_fabricapi.py │ ├── test_filesystem.py │ ├── test_local_config_util.py │ ├── test_parser.py │ ├── test_remote_config_util.py │ ├── test_validators.py │ └── test_version_util.py │ └── yarn_slider │ ├── __init__.py │ └── test_help.py ├── tox.ini └── util ├── __init__.py ├── http.py └── semantic_version.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | *.rpm 4 | *.egg 5 | *.yaml 6 | *.bak 7 | prestoadmin.egg-info/ 8 | .tox/ 9 | .coverage 10 | htmlcov/ 11 | log/ 12 | tmp/ 13 | 14 | # Ignore generated Sphinx docs 15 | docs/prestoadmin.* 16 | docs/modules.rst 17 | docs/_build 18 | 19 | # Ignore build folders 20 | build/ 21 | dist/ 22 | .eggs/ 23 | .idea/ 24 | *.iml 25 | *.egg/ 26 | 27 | # tmp backup files 28 | *~ 29 | \#*# 30 | .#* 31 | 32 | #mvn targets 33 | presto-admin-test/target 34 | 35 | # presto yarn package for product tests 36 | presto-yarn-package.zip 37 | -------------------------------------------------------------------------------- /.pypirc: -------------------------------------------------------------------------------- 1 | [distutils] 2 | index-servers = 3 | pypitest 4 | pypi 5 | 6 | [pypitest] 7 | repository: https://testpypi.python.org/pypi 8 | 9 | [pypi] 10 | repository: https://pypi.python.org/pypi 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: "2.7" 3 | sudo: required 4 | group: deprecated-2017Q2 5 | dist: trusty 6 | services: 7 | - docker 8 | env: 9 | global: 10 | - PA_TEST_ONLINE_INSTALLER=true 11 | - PYTHONPATH=$PYTHONPATH:$(pwd) 12 | - LONG_PRODUCT_TESTS="tests/product/test_server_install.py tests/product/test_status.py tests/product/test_collect.py tests/product/test_catalog.py tests/product/test_control.py tests/product/test_server_uninstall.py" 13 | matrix: 14 | - OTHER_TESTS=true 15 | - SHORT_PRODUCT_TEST_GROUP=0 16 | - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN="tests/product/test_server_install.py" 17 | - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_status.py" 18 | - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_collect.py tests/product/test_server_uninstall.py" 19 | - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_catalog.py" 20 | - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_control.py" 21 | install: 22 | - pip install --upgrade pip==9.0.1 23 | - pip install -r requirements.txt 24 | - pip install tox-travis 25 | before_script: 26 | - make docker-images 27 | - make presto-server-rpm.rpm 28 | script: 29 | - | 30 | if [ -v SHORT_PRODUCT_TEST_GROUP ]; then 31 | ALL_PRODUCT_TESTS=$(find tests/product/ -name 'test_*py' | grep -v __init__ | xargs wc -l | sort -n | head -n -1 | awk '{print $2}' | tr '\n' ' ') 32 | for LONG_PRODUCT_TEST in ${LONG_PRODUCT_TESTS[@]}; do 33 | ALL_PRODUCT_TESTS=${ALL_PRODUCT_TESTS//$LONG_PRODUCT_TEST/}; 34 | if [ $? -ne 0 ]; then 35 | exit 1 36 | fi 37 | done 38 | SHORT_PRODUCT_TESTS=$(echo $ALL_PRODUCT_TESTS | tr ' ' '\n' | awk "NR % 1 == $SHORT_PRODUCT_TEST_GROUP" | tr '\n' ' ') 39 | ./bin/ci-product.sh ${SHORT_PRODUCT_TESTS}; 40 | elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN ]; then 41 | export IMAGE_NAMES="standalone_presto_admin" 42 | ./bin/ci-product.sh ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN}; 43 | elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO ]; then 44 | export IMAGE_NAMES="standalone_presto standalone_presto_admin" 45 | ./bin/ci-product.sh ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO} 46 | elif [ -v OTHER_TESTS ]; then 47 | ./bin/ci-basic.sh 48 | else 49 | echo "Unknown test" 50 | exit 1 51 | fi 52 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/prestodb/presto-admin/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Your operating system name and version. 21 | * Any details about your local setup that might be helpful in troubleshooting. 22 | * Detailed steps to reproduce the bug. 23 | 24 | Fix Bugs 25 | ~~~~~~~~ 26 | 27 | Look through the GitHub issues for bugs. Anything tagged with "bug" 28 | is open to whomever wants to implement it. 29 | 30 | Implement Features 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | Look through the GitHub issues for features. Anything tagged with "feature" 34 | is open to whomever wants to implement it. 35 | 36 | Write Documentation 37 | ~~~~~~~~~~~~~~~~~~~ 38 | 39 | presto-admin could always use more documentation, whether as part of the 40 | official presto-admin docs, in docstrings, or even on the web in blog posts, 41 | articles, and such. 42 | 43 | Submit Feedback 44 | ~~~~~~~~~~~~~~~ 45 | 46 | The best way to send feedback is to file an issue at https://github.com/prestodb/presto-admin/issues. 47 | 48 | If you are proposing a feature: 49 | 50 | * Explain in detail how it would work. 51 | * Keep the scope as narrow as possible, to make it easier to implement. 52 | 53 | Get Started! 54 | ------------ 55 | 56 | Ready to contribute? Here's how to set up `presto-admin` for local development. 57 | 58 | 1. Fork the `presto-admin` repo on GitHub, https://github.com/prestodb/presto-admin. 59 | 2. Clone your fork locally:: 60 | 61 | $ git clone git@github.com:your_name_here/presto-admin.git 62 | 63 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 64 | 65 | $ mkvirtualenv prestoadmin 66 | $ cd prestoadmin/ 67 | $ python setup.py develop 68 | 69 | 4. Create a branch for local development:: 70 | 71 | $ git checkout -b name-of-your-bugfix-or-feature 72 | 73 | Now you can make your changes locally. 74 | 75 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox. 76 | To run tests, you need docker installed. You may also need to pip install wheel into your virtualenv. To install and start docker use:: 77 | 78 | $ wget -qO- https://get.docker.com/ | sh 79 | 80 | # Add current user to Docker group to run without sudo 81 | $ sudo gpasswd -a ${USER} docker 82 | $ sudo service docker restart 83 | 84 | Now, to run presto-admin tests:: 85 | 86 | $ make lint 87 | $ make test-all 88 | 89 | 6. Commit your changes and push your branch to GitHub:: 90 | 91 | $ git add . 92 | $ git commit -m "Your detailed description of your changes." 93 | $ git push origin name-of-your-bugfix-or-feature 94 | 95 | 7. Submit a pull request through the GitHub website. 96 | 97 | Pull Request Guidelines 98 | ----------------------- 99 | 100 | Before you submit a pull request, check that it meets these guidelines: 101 | 102 | 1. The pull request should include tests. 103 | 2. If the pull request adds functionality, the docs should be updated. Put 104 | your new functionality into a function with a docstring, and add the 105 | feature to the presto-admin/docs. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTING.rst 2 | include HISTORY.rst 3 | include LICENSE 4 | include README.md 5 | 6 | recursive-include tests * 7 | recursive-exclude * __pycache__ 8 | recursive-exclude * *.py[co] 9 | recursive-include prestoadmin *.ini 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # presto-admin [![Build Status](https://travis-ci.org/prestodb/presto-admin.svg?branch=master)](https://travis-ci.org/prestodb/presto-admin) 2 | 3 | presto-admin installs, configures, and manages Presto installations. 4 | 5 | Comprehensive documentation can be found [here](http://prestodb.github.io/presto-admin/). 6 | 7 | ## Requirements 8 | 9 | 1. Python 2.6 or 2.7 10 | 2. [Docker](https://www.docker.com/). (Only required for development, if you want to run the system tests) 11 | * If you DO NOT have Docker already installed, you can run the `install-docker.sh` 12 | script in the `bin` directory of this project. That script has only been tested on 13 | Ubuntu 14.04. 14 | * If you have Docker already installed, you need to make sure that your user has 15 | been added to the docker group. This will enable you to run commands without `sudo`, 16 | which is a requirement for some of the unit tests. To enable sudoless docker access 17 | run the following: 18 | 19 | $ sudo groupadd docker 20 | $ sudo gpasswd -a ${USER} docker 21 | $ sudo service docker restart 22 | 23 | If the user you added to the docker group is the same one you're logged in as, you will 24 | need to log out and back in so that the changes can take effect. 25 | 26 | ## Building 27 | 28 | Presto-admin makes use of `make` as its build tool. `make` in turn calls out to various utilities (e.g. 29 | `tox`, `flake8`, `sphinx-apidoc`, `python`) in order to perform the requested actions. 30 | 31 | In order to get started with `presto-admin`, 32 | 33 | 1. Fork the `presto-admin` repo on GitHub, https://github.com/prestodb/presto-admin. 34 | 2. Clone your fork locally :: 35 | 36 | $ git clone git@github.com:your_name_here/presto-admin.git 37 | 38 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development :: 39 | 40 | $ mkvirtualenv prestoadmin 41 | $ cd prestoadmin/ 42 | $ python setup.py develop 43 | 44 | 4. Create a branch for local development :: 45 | 46 | $ git checkout -b name-of-your-bugfix-or-feature 47 | 48 | Now you can make your changes locally. 49 | 50 | 5. When you're done making changes, check that your changes pass `make clean lint test`, which runs flake8 and the unit tests (which test both Python 2.6 and 2.7). 51 | To run the product tests tests (`make test-all`), you need docker installed. You may also need to run `pip install wheel` in your virtualenv. To install and start docker use :: 52 | 53 | $ wget -qO- https://get.docker.com/ | sh 54 | 55 | # Add current user to Docker group to run without sudo 56 | $ sudo gpasswd -a ${USER} docker 57 | $ sudo service docker restart 58 | 59 | 60 | ### Building the installer 61 | 62 | The two tasks used to build the presto-admin installer are `dist` and 63 | `dist-offline`. The `dist` task builds an installer that requires internet 64 | connectivity during installation. The `dist-offline` task builds an installer 65 | that does not require internet connectivity during installation. Instead the 66 | offline installer downloads all dependencies at build time and points `pip` to 67 | those dependencies during installation. 68 | 69 | ## License 70 | 71 | Free software: Apache License Version 2.0 (APLv2). 72 | -------------------------------------------------------------------------------- /base-images-tag.json: -------------------------------------------------------------------------------- 1 | { 2 | "base_images_tag": "7" 3 | } 4 | -------------------------------------------------------------------------------- /bin/ci-basic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | make clean lint dist docs 16 | tox -e py26 -- -s tests.unit 17 | tox -e py26 -- -s tests.integration 18 | tox -e py27 -- -s tests.unit 19 | tox -e py27 -- -s tests.integration 20 | -------------------------------------------------------------------------------- /bin/ci-product.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | make test-images 16 | nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' "$@" 17 | -------------------------------------------------------------------------------- /bin/install-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Install docker on Ubuntu 14.04 16 | wget -qO- https://get.docker.com/ | sh 17 | 18 | # Add current user to Docker group to run without sudo 19 | sudo gpasswd -a ${USER} docker 20 | 21 | sudo sh -c "echo 'DOCKER_OPTS=\"--dns 153.65.2.111 --dns 8.8.8.8\"' >> /etc/default/docker" 22 | 23 | sudo service docker restart 24 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/emr.rst: -------------------------------------------------------------------------------- 1 | .. _presto-admin-on-emr-label: 2 | .. 3 | .. If you modify this file, you will have to modify the NOTEs in the following files: 4 | .. docs/installation/java-installation.rst 5 | .. docs/installation/presto-admin-configuration.rst 6 | .. docs/installation/presto-admin-installation.rst 7 | .. 8 | 9 | ================================================ 10 | Setting up Presto Admin on an Amazon EMR cluster 11 | ================================================ 12 | 13 | To install, configure and run Presto Admin on an Amazon EMR cluster, follow the instructions in :ref:`quick-start-guide-label`, but pay attention to the notes or sections specfic to EMR cluster. We reiterate these EMR specific caveats below: 14 | 15 | - To install Presto Admin on an Amazon EMR cluster, follow the instructions in :ref:`presto-admin-installation-label` except for the following difference: 16 | 17 | - Use the online installer instead of the offline installer (see explanation :ref:`presto-admin-installation-label`). 18 | 19 | - To configure Presto Admin on an Amazon EMR cluster, follow the instructions in :ref:`presto-admin-configuration-label`. Specifically, we recommend the following property values during the configuration: 20 | 21 | - Use ``hadoop`` as the ``username`` instead of the default username ``root`` in the ``config.json`` file. 22 | 23 | - Use the host name of the EMR master node as the ``coordinator`` in the ``config.json`` file. 24 | 25 | - To run Presto Admin on EMR, see the sections starting from :ref:`presto-server-installation-label` onwards in :ref:`quick-start-guide-label`) except for the following caveats: 26 | 27 | - The default version of Java installed on an EMR cluster (up to EMR 4.4.0) is 1.7, whereas Presto requires Java 1.8. Install Java 1.8 on the EMR cluster by following the instructions in :ref:`java-installation-label`. 28 | 29 | - For running Presto Admin commands on an EMR cluster, do the following: 30 | * Copy the ``.pem`` file associated with the Amazon EC2 key pair to the Presto Admin installation node of the EMR cluster. 31 | * Use the ``-i `` input argument when running presto-admin commands on the node. 32 | :: 33 | 34 | -i 35 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Presto-Admin 2 | ============ 3 | 4 | `Mailing list `_ | 5 | `Issues `_ | 6 | `Github `_ | 7 | 8 | Introduction 9 | ------------ 10 | Presto-Admin is a tool for installing and managing the Presto query engine on a 11 | cluster. It provides easy-to-use commands to: 12 | 13 | * Install and uninstall Presto across your cluster 14 | * Configure your Presto cluster 15 | * Start and stop the Presto servers 16 | * Gather status and log information from your Presto cluster 17 | 18 | Content 19 | ------- 20 | 21 | .. toctree:: 22 | :maxdepth: 3 23 | 24 | software-requirements 25 | user-guide 26 | contributing 27 | release 28 | 29 | .. toctree:: 30 | :hidden: 31 | 32 | modules 33 | 34 | Indices and tables 35 | ------------------ 36 | 37 | * :ref:`search` 38 | 39 | -------------------------------------------------------------------------------- /docs/installation/advanced-installation-options.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | Advanced Installation Options 3 | ============================= 4 | 5 | Specifying a Certificate Authority for the Online Installer 6 | ----------------------------------------------------------- 7 | The online installer downloads its dependencies from ``pypi.python.org``, the 8 | standard Python location for hosting packages. For some operating systems, 9 | the certificate for pypi.python.org is not included in the CA cert bundle, 10 | so our installation scripts specify ``--trusted-host pypi.python.org`` when 11 | downloading the dependencies. 12 | 13 | If using ``--trusted-host`` is not suitable for your security needs, it is 14 | possible to supply your own certificates to use to authenticate to 15 | ``pypi.python.org``. Please note that if these certificates do not work to 16 | access ``pypi.python.org``, the installation will fail. For example, to install 17 | with your own certificates: 18 | 19 | :: 20 | 21 | ./install-prestoadmin.sh /cacert.pem 22 | 23 | Coordinator failover 24 | -------------------- 25 | Presto does not yet support automatic failover for the coordinator. You can 26 | migrate to a new coordinator using the ``presto-admin`` -H and -x flags 27 | to include and exclude hosts in your command, respectively. 28 | 29 | To view these ``presto-admin`` options, use the ``--extended-help`` flag. 30 | 31 | You can switch to a new coordinator by following the steps below: 32 | 33 | 1. Stop Presto on all the nodes where it is running using the command: :: 34 | 35 | ./presto-admin server stop 36 | 37 | 2. Edit the ``presto-admin`` topology file and replace the old coordinator 38 | with the new one. By default, the topology file is located at 39 | ``~/.prestoadmin/config.json``. 40 | 41 | 3. To install Presto on the new node, run the following two ``presto-admin`` 42 | commands. The first command is needed only if Java 8 is not already installed 43 | on the new coordinator: :: 44 | 45 | ./presto-admin package install -H new_coordinator /path/to/jdk8.rpm 46 | ./presto-admin server install -H new_coordinator /path/to/presto-server.rpm 47 | 48 | 4. Update the coordinator and worker configuration files controlled by 49 | ``presto-admin``. By default, these files are available at ``~/.prestoadmin/``. 50 | 51 | 5. Run the following commands to deploy the new configurations to all nodes, 52 | including the new coordinator and start the server: :: 53 | 54 | ./presto-admin configuration deploy 55 | ./presto-admin server start 56 | -------------------------------------------------------------------------------- /docs/installation/java-installation.rst: -------------------------------------------------------------------------------- 1 | .. _java-installation-label: 2 | 3 | ================= 4 | Installing Java 8 5 | ================= 6 | Prerequisites: :ref:`presto-admin-installation-label` and :ref:`presto-admin-configuration-label` 7 | 8 | The Oracle Java 1.8 JRE (64-bit), update 45 or higher, is a prerequisite for Presto. If a suitable 64-bit version of Oracle Java 8 is already installed on the cluster, you can skip this step. 9 | 10 | .. NOTE:: On Amazon EMR (up to EMR 4.4.0), the default version of Java is 1.7. To run Presto on EMR, please install Java 1.8. 11 | 12 | There are two ways to install Java: via RPM and via tarball. The RPM installation sets the default Java on your machine to be Java 8. If 13 | it is acceptable to set the default Java to be Java 8, you can use ``presto-admin`` to install Java, otherwise you will need to install Java 8 manually. 14 | 15 | To install Java via RPM using ``presto-admin``: 16 | 17 | 1. Download `Oracle Java 8 `_, selecting the Oracle Java 1.8 (64-bit) RPM download for Linux. 18 | 19 | 2. Copy the RPM to a location accessible by ``presto-admin``. 20 | 21 | 3. Run the following command to install Java 8 on each node in the Presto cluster: :: 22 | 23 | $ ./presto-admin package install 24 | 25 | 26 | .. NOTE:: The ``server-install-label`` will look for your Oracle Java 1.8 installation at locations where Java is normally installed when using the binary or the RPM based installer. Otherwise, you need to have an environment variable called ``JAVA8_HOME`` set with your Java 1.8 install path. If ``JAVA8_HOME`` is not set or is pointing to an incompatible version of Java, the installer will look for the ``JAVA_HOME`` environment variable for a compatible version of Java. If neither of these environmental variables is set with a compatible version, and ``presto-admin`` fails to find Java 8 at any of the normal installation locations, then ``server install`` will fail. After successfully running ``server install`` you can find the Java being used by Presto at ``/etc/presto/env.sh``. 27 | 28 | .. NOTE:: If you have installed the JDK, ``JAVA8_HOME`` should be set so refer to the ``jre`` subdirectory of the JDK. 29 | 30 | .. NOTE:: If installing Java on SLES, you will need to specify the flag ``--nodeps`` for ``presto-admin package install``, so that the RPM is installed without checking or validating dependencies. 31 | -------------------------------------------------------------------------------- /docs/installation/presto-admin-installation.rst: -------------------------------------------------------------------------------- 1 | .. _presto-admin-installation-label: 2 | 3 | ======================= 4 | Installing Presto Admin 5 | ======================= 6 | 7 | Prerequisites: 8 | - `Python 2.6 or Python 2.7 `_. 9 | - If you are using the online installer then make sure you've installed the 10 | Python development package for your system. For RedHat/Centos that package is 11 | ``python2-devel`` and for Debian/Ubuntu it is ``python-dev``. 12 | 13 | Presto Admin is packaged as an offline installer -- 14 | ``prestoadmin--offline.tar.gz`` -- and as an online 15 | installer -- ``prestoadmin--online.tar.gz``. 16 | 17 | The offline installer includes all of the dependencies for 18 | ``presto-admin``, so it can be used on a cluster without an outside 19 | network connection. The offline installer is currently only supported 20 | on RedHat Linux 6.x or CentOS equivalent. 21 | 22 | The online installer downloads all of the dependencies when you run 23 | ``./install-prestoadmin.sh``. You must use the online installer for 24 | installation of Presto on Amazon EMR and for use on any operating 25 | system not listed above. If you are using presto-admin on an 26 | unsupported operating system, there may be operating system 27 | dependencies beyond the installation process, and presto-admin may not 28 | work. 29 | 30 | To install ``presto-admin``: 31 | 32 | 1. Copy the installer ``prestoadmin--offline.tar.gz`` to the 33 | location where you want ``presto-admin`` to run. 34 | Note that ``presto-admin`` does not have to be on the same node(s) 35 | where Presto will run, though it does need to have SSH access to all 36 | of the nodes in the cluster. 37 | 38 | .. NOTE:: 39 | For Amazon EMR, use the online installer instead of the offline installer. 40 | 41 | 2. Extract and run the installation script from within the ``prestoadmin`` directory. 42 | :: 43 | 44 | $ tar xvf prestoadmin--offline.tar.gz 45 | $ cd prestoadmin 46 | $ ./install-prestoadmin.sh 47 | 48 | The installation script will create a ``presto-admin-install`` directory and an 49 | executable ``presto-admin`` script. By default, the ``presto-admin`` config and log 50 | directory locations are configured to be ``~/.prestoadmin`` and ``~/.prestoadmin/log``, 51 | respectively. This can be changed by modifying the environment variables, 52 | PRESTO_ADMIN_CONFIG_DIR and PRESTO_ADMIN_LOG_DIR. The installation script will also create 53 | the directories pointed to by PRESTO_ADMIN_CONFIG_DIR and PRESTO_ADMIN_LOG_DIR. If those 54 | directories already exist, the installation script will not erase their contents. 55 | 56 | 3. Verify that ``presto-admin`` was installed properly by running the following 57 | command: 58 | :: 59 | 60 | $ ./presto-admin --help 61 | 62 | Please note that you should only run one ``presto-admin`` command on your 63 | cluster at a time. 64 | -------------------------------------------------------------------------------- /docs/installation/presto-admin-upgrade.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Upgrading Presto-Admin 3 | ====================== 4 | 5 | Upgrading to a newer version of ``presto-admin`` requires deleting the old 6 | installation and then installing the new version. After you've deleted the 7 | ``prestoadmin`` directory, install the newer version of ``presto-admin`` 8 | by following the instructions in the installation section 9 | (see :ref:`presto-admin-installation-label`). 10 | 11 | For ``presto-admin`` versions earlier than 2.0, the configuration files are 12 | located at ``/etc/opt/prestoadmin``. To upgrade to a newer version and 13 | continue to use these configuration files, make sure you copy them to the 14 | new configuration directory at ``~/.prestoadmin`` (or 15 | ``$PRESTO_ADMIN_CONFIG_DIR``). The connector configuration directory 16 | located at ``/etc/opt/prestoadmin/connectors`` must be renamed to 17 | ``/etc/opt/prestoadmin/catalog``, before copying to ``~/.prestoadmin``. 18 | 19 | For ``presto-admin`` versions 2.0 and later, the configuration files 20 | located in ``~/.prestoadmin`` will remain intact and continue to be used 21 | by the newer version of ``presto-admin``. 22 | -------------------------------------------------------------------------------- /docs/installation/presto-catalog-installation.rst: -------------------------------------------------------------------------------- 1 | 2 | ================ 3 | Adding a Catalog 4 | ================ 5 | 6 | In Presto, connectors allow you to access different data sources -- e.g., 7 | Hive, PostgreSQL, or MySQL. 8 | 9 | To add a catalog for the Hive connector: 10 | 11 | 1. Create a file ``hive.properties`` in ``~/.prestoadmin/catalog`` with the following content: :: 12 | 13 | connector.name=hive-hadoop2 14 | hive.metastore.uri=thrift://: 15 | 16 | 17 | 2. Distribute the configuration file to all of the nodes in the cluster: :: 18 | 19 | ./presto-admin catalog add hive 20 | 21 | 22 | 3. Restart Presto: :: 23 | 24 | ./presto-admin server restart 25 | 26 | 27 | You may need to add additional properties for the Hive connector to work properly, such as if your Hadoop cluster 28 | is set up for high availability. For these and other properties, see the `Hive connector documentation `_. 29 | 30 | For detailed documentation on ``catalog add``, see :ref:`catalog-add`. 31 | For more on which catalogs Presto supports, see the `Presto connector documentation `_. 32 | -------------------------------------------------------------------------------- /docs/installation/presto-cli-installation.rst: -------------------------------------------------------------------------------- 1 | .. _presto-cli-installation-label: 2 | 3 | ====================== 4 | Running Presto Queries 5 | ====================== 6 | 7 | The Presto CLI provides a terminal-based interactive shell for running queries. The CLI is a self-executing JAR file, which means it acts like a normal UNIX executable. 8 | 9 | To run a query via the Presto CLI: 10 | 11 | 1. Download the ``presto-cli`` and copy it to the location you want to run it from. This location may be any node that has network access to the coordinator. 12 | 13 | 2. Rename the artifact to ``presto`` and make it executable, substituting your version of Presto for "version": :: 14 | 15 | $ mv presto-cli--executable.jar presto 16 | $ chmod +x presto 17 | 18 | .. NOTE:: Presto must run with Java 8, so if Java 7 is the default on your cluster, you will need to explicitly specify the Java 8 executable. For example, `` -jar presto``. It may be helpful to add an alias for the Presto CLI: ``alias presto=' -jar '``. 19 | 20 | 3. By default, ``presto-admin`` configures a TPC-H catalog, which generates TPC-H data on-the-fly. 21 | Using this catalog, issue the following commands to run your first Presto query: :: 22 | 23 | $ ./presto --catalog tpch --schema tiny 24 | $ select count(*) from lineitem; 25 | 26 | 27 | The above command assumes that you installed the Presto CLI on the coordinator, and that the Presto server is on port 8080. If either of these are not the case, then specify the server location in the command: :: 28 | 29 | $ ./presto --server : --catalog tpch --schema tiny 30 | 31 | -------------------------------------------------------------------------------- /docs/installation/presto-port-configuration.rst: -------------------------------------------------------------------------------- 1 | .. _presto-port-configuration-label: 2 | 3 | =========================== 4 | Configuring the Presto Port 5 | =========================== 6 | 7 | By default, Presto uses 8080 for the HTTP port. If the port is already in use on any given node on your cluster, Presto will not start on that node(s). 8 | 9 | To configure the server to use a different port: 10 | 11 | 1. Select a port that is free on all of the nodes. You can check if a port is already in use on a node by running the following on that node: 12 | :: 13 | 14 | netstat -an |grep 8081 |grep LISTEN 15 | 16 | It will return nothing if port 8081 is free. 17 | 18 | 2. Modify the following properties in ``~/.prestoadmin/coordinator/config.properties`` and ``~/.prestoadmin/workers/config.properties``: 19 | 20 | :: 21 | 22 | http-server.http.port= 23 | discovery.uri=http://: 24 | 25 | 26 | 3. Run the following command to deploy the configuration change to the cluster: :: 27 | 28 | ./presto-admin configuration deploy 29 | 30 | 31 | 4. Restart the Presto servers so that the changes get picked up: :: 32 | 33 | ./presto-admin server restart 34 | -------------------------------------------------------------------------------- /docs/installation/presto-server-installation.rst: -------------------------------------------------------------------------------- 1 | .. _presto-server-installation-label: 2 | 3 | ============================ 4 | Installing the Presto Server 5 | ============================ 6 | Prerequisites: :ref:`presto-admin-installation-label`, :ref:`java-installation-label` and :ref:`presto-admin-configuration-label` 7 | 8 | To install the Presto query engine on a cluster of nodes using ``presto-admin``: 9 | 10 | 1. Download ``presto-server-rpm-VERSION.ARCH.rpm`` 11 | 12 | 2. Copy the RPM to a location accessible by ``presto-admin``. 13 | 14 | 3. Run the following command to install Presto: :: 15 | 16 | $ ./presto-admin server install 17 | 18 | 19 | Presto! Presto is now installed on the coordinator and workers specified in your ``~/.prestoadmin/config.json`` file. 20 | 21 | The default port for Presto is 8080. If that port is already in use on your cluster, you will not be able to start Presto. 22 | In order to change the port that Presto uses, proceed to :ref:`presto-port-configuration-label`. 23 | 24 | There are additional configuration properties described at :ref:`presto-configuration-label` that 25 | must be changed for optimal performance. These configuration changes can be done either 26 | before or after starting the Presto server and running queries for the first time, though 27 | all configuration changes require a restart of the Presto servers. 28 | 29 | 4. Now, you are ready to start Presto: :: 30 | 31 | $ ./presto-admin server start 32 | 33 | This may take a few seconds, since the command doesn't exit until ``presto-admin`` verifies that Presto is fully up and ready to receive queries. 34 | -------------------------------------------------------------------------------- /docs/installation/troubleshooting-installation.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Troubleshooting 3 | =============== 4 | 5 | #. To troubleshoot problems with presto-admin or Presto, you can use the 6 | incident report gathering commands from presto-admin to gather logs and 7 | other system information from your cluster. Relevant commands: 8 | 9 | * :ref:`collect-logs` 10 | * :ref:`collect-query-info` 11 | * :ref:`collect-system-info` 12 | 13 | #. You can find the ``presto-admin`` logs in the ``~/.prestoadmin/log`` 14 | directory. 15 | #. You can check the status of Presto on your cluster by using 16 | :ref:`server-status`. 17 | #. If Presto is not running and you try to execute any command from the Presto CLI you might get: 18 | :: 19 | 20 | $ Error running command: Server refused connection: http://localhost:8080/v1/statement 21 | 22 | To fix this, start Presto with: 23 | :: 24 | 25 | $ ./presto-admin server start 26 | 27 | #. If the Presto servers fail to start or crash soon after starting, look at 28 | the presto server logs on the Presto cluster ``/var/log/presto`` for an 29 | error message. You can collect the logs locally using :ref:`collect-logs`. 30 | The relevant error messages should be at the end of the log with the most 31 | recent timestamp. Below are tips for some common errors: 32 | 33 | * Specifying a port that is already in use: Look at 34 | :ref:`presto-port-configuration-label` to learn how to change the port 35 | configuration. 36 | * An error in a catalog configuration file, such as a syntax error or 37 | a missing connector.name property: correct the file and deploy it to the 38 | cluster again using :ref:`catalog-add` 39 | 40 | #. The following error can occur if you do not have passwordless ssh enabled 41 | and have not provided a password or if the user requires a sudo password: :: 42 | 43 | Fatal error: Needed to prompt for a connection or sudo password (host: master), but input would be ambiguous in parallel mode 44 | 45 | See :ref:`ssh-configuration-label` for information on setting up 46 | passwordless ssh and on providing a password, and :ref:`sudo-password-spec` 47 | for information on providing a sudo password. 48 | 49 | #. Support for connecting to a cluster with internal HTTPS and/or LDAP communication 50 | enabled is experimental. Make sure to check both the Presto server log and the 51 | ``presto-admin`` log to troubleshoot problems with your configuration; it may also 52 | be helpful to verify that you can connect to the cluster via the Presto CLI using 53 | HTTPS/LDAP as appropriate. 54 | -------------------------------------------------------------------------------- /docs/quick-start-guide.rst: -------------------------------------------------------------------------------- 1 | .. _quick-start-guide-label: 2 | 3 | ***************** 4 | Quick Start Guide 5 | ***************** 6 | 7 | The following describes installing Presto on one or more nodes via the ``presto-admin`` software. This is an alternative to the installation steps described at `prestodb.io `_. Using the ``presto-admin`` tool is the simplest and preferred method for installing and managing a Presto cluster. 8 | 9 | For a detailed explanation of all of the commands and their options, see :ref:`comprehensive-guide-label`. 10 | 11 | .. toctree:: 12 | :maxdepth: 1 13 | 14 | installation/presto-admin-installation 15 | installation/presto-admin-configuration 16 | installation/java-installation 17 | installation/presto-server-installation 18 | installation/presto-cli-installation 19 | installation/presto-catalog-installation 20 | installation/presto-configuration 21 | installation/troubleshooting-installation 22 | installation/presto-admin-upgrade 23 | -------------------------------------------------------------------------------- /docs/release.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Release Notes 3 | ============= 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | release/release-2.2 8 | release/release-2.1 9 | release/release-2.0 10 | release/release-1.5 11 | release/release-1.4 12 | release/release-1.3 13 | release/release-1.2 14 | release/release-1.1 15 | release/release-0.1.0 16 | -------------------------------------------------------------------------------- /docs/release/release-0.1.0.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Release 0.1.0 3 | ============= 4 | 5 | Initial Release! 6 | This release works for Presto versions 0.100-0.102 7 | -------------------------------------------------------------------------------- /docs/release/release-1.1.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 1.1 3 | =========== 4 | 5 | This release works for Presto versions 0.103-0.115 6 | -------------------------------------------------------------------------------- /docs/release/release-1.2.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 1.2 3 | =========== 4 | 5 | The default values in this release are intended to work with Presto versions 6 | 0.116 through at least 0.130. However, the user can supply non-default 7 | configurations to use this release with other versions of Presto. 8 | 9 | General Fixes 10 | ------------- 11 | * Fix server status to work with later versions of Presto 12 | * Exit with non-zero code when operations fail 13 | * Update configuration defaults for Presto versions >0.115 14 | * Make remote log directory configurable 15 | * Add support for specifying java8 home in config.json 16 | * :ref:`collect-logs` will use the log directory specified in 17 | Presto's config.properties if configured. 18 | 19 | 20 | Configuration 21 | ------------- 22 | Before this release, :ref:`configuration-deploy-label` would fill in default 23 | values for any required properties that the user did not supply in the 24 | configuration files. However, this created problems when different versions of 25 | Presto had different configuration requirements. In particular, it became 26 | impossible to remove any required properties from the configuration even if the 27 | user's Presto version did not require those properties. 28 | 29 | In the current behavior, when the user needs to override the defaults in any 30 | configuration file, they must write out all the properties for that 31 | configuration file, which will be deployed as-is. 32 | -------------------------------------------------------------------------------- /docs/release/release-1.3.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 1.3 3 | =========== 4 | 5 | The default values in this release are intended to work with Presto versions 6 | 0.116 through x. However, the user can supply non-default 7 | configurations to use this release with other versions of Presto. 8 | 9 | General Fixes 10 | ------------- 11 | * Change ``make dist`` to build the online installer by default 12 | -------------------------------------------------------------------------------- /docs/release/release-1.4.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 1.4 3 | =========== 4 | 5 | This release works for Presto versions 0.116-0.148. 6 | 7 | * Add package uninstall support 8 | * Add --nodeps option to indicate if the server install/uninstall should ignore dependencies 9 | * Fix config files to be owned by the presto user and not accessible to other users 10 | * Update and add more Presto configuration defaults 11 | * Use proper Java version for server upgrade 12 | 13 | -------------------------------------------------------------------------------- /docs/release/release-1.5.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 1.5 3 | =========== 4 | 5 | This release works for Presto versions 0.116 through at least 0.152.1 6 | 7 | New Features 8 | ------------ 9 | * Add the ability to download the rpm in ``server install`` by specifying ``latest`` or a version number 10 | * Add a ``file copy`` command to distribute files to all nodes on the cluster 11 | * Collect connector configurations from each node as part of ``collect system_info`` 12 | 13 | Bug Fixes 14 | --------- 15 | * Fix a bug where a non-root user in ``config.json`` could not access files 16 | 17 | Compatiblity Notes 18 | ------------------ 19 | * The ``script run`` command was renamed to ``file run`` 20 | -------------------------------------------------------------------------------- /docs/release/release-2.0.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 2.0 3 | =========== 4 | 5 | New Features 6 | ------------ 7 | * Make presto-admin log and configuration directories configurable. They can be 8 | set using the environment variables ``PRESTO_ADMIN_LOG_DIR`` and 9 | ``PRESTO_ADMIN_CONFIG_DIR``. 10 | * Change the default configuration directory to ``~/.prestoadmin`` and the 11 | default log directory to ``~/.prestoadmin/log``. 12 | * Remove the requirement for running and installing presto-admin with sudo. 13 | The user specified in ``config.json`` still needs sudo access on the Presto 14 | nodes in order to execute commands like installing the RPM and setting 15 | permissions on the configuration files. 16 | * Rename the ``connectors`` directory to ``catalog`` to match the Presto 17 | nomenclature. 18 | * Rename the ``connector add`` and ``connector remove``. commands to 19 | ``catalog add`` and ``catalog remove``. 20 | * Add experimental support for connecting to a Presto server with internal 21 | communication via HTTPS and LDAP, where the HTTP connection is disabled. 22 | * Allow specifying which python interpreter to use as an argument to the 23 | presto-admin installation script. 24 | * Add ``G1HeapRegionSize=32M`` to the jvm.config defaults as suggested by the 25 | Presto documentation. 26 | 27 | Bug Fixes 28 | --------- 29 | * Keep the ``node.id`` in Presto's ``node.properties`` file consistent across 30 | configuration updates. 31 | * Change the permissions on the Presto catalog directory to ``755`` and the 32 | owner to``presto:presto``. 33 | * Use ``catalog.config-dir`` instead of ``plugin.config-dir`` in the 34 | ``node.properties`` defaults. ``plugin.config-dir`` has been deprecated 35 | in Presto since version 0.113. 36 | 37 | Compatibility Notes 38 | ------------------- 39 | * The locations of config and log directories have been changed 40 | * The ``connectors`` directory has been renamed to ``catalog``. 41 | * The ``connector`` commands have been renamed to ``catalog``. 42 | -------------------------------------------------------------------------------- /docs/release/release-2.1.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 2.1 3 | =========== 4 | 5 | Bug Fixes 6 | --------- 7 | * Fix bug with ``server start`` when only frontend LDAP in Presto is enabled. 8 | * Fix intermittent bug with ``server start`` printing out irrelevant error messages. 9 | -------------------------------------------------------------------------------- /docs/release/release-2.2.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Release 2.2 3 | =========== 4 | 5 | New Features 6 | ------------ 7 | * Support specifying a range of workers in ``config.json`` 8 | 9 | Bug Fixes and Enhancements 10 | -------------------------- 11 | * Fix error with getting server status for complex Presto version names 12 | * Preserve all of ``/etc/presto`` during upgrade 13 | * Use ``rpm -U`` for ``package upgrade`` and ``server upgrade`` instead of uninstalling and reinstalling fresh 14 | * Use ``.gz`` instead of ``.bz2`` for the installation tarballs and for the files collected by ``collect logs`` 15 | and ``collect system_info`` 16 | 17 | -------------------------------------------------------------------------------- /docs/software-requirements.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Software Requirements 3 | ===================== 4 | 5 | **Operating Systems** 6 | * RedHat Linux version 6.x 7 | * CentOS (equivalent to above) 8 | 9 | **Python** 10 | 11 | * Python 2.6.x OR 12 | * Python 2.7.x 13 | 14 | **SSH Configuration** 15 | 16 | * Passwordless SSH from the node running ``presto-admin`` to the nodes where Presto will be installed OR 17 | * Ability to SSH with a password from the node running ``presto-admin`` to the nodes where Presto will be installed 18 | 19 | For more on SSH configuration, see :ref:`ssh-configuration-label`. 20 | 21 | **Other Configuration** 22 | 23 | * Sudo privileges on both the node running ``presto-admin`` and the nodes where Presto will be installed are required for a non-root presto-admin user. 24 | -------------------------------------------------------------------------------- /docs/ssh-configuration.rst: -------------------------------------------------------------------------------- 1 | .. _ssh-configuration-label: 2 | 3 | ***************** 4 | SSH Configuration 5 | ***************** 6 | 7 | In order to run ``presto-admin``, the node that is running ``presto-admin`` must be able to connect to all of the nodes running Presto via SSH. ``presto-admin`` makes the SSH connection with the username and port specified in ``~/.prestoadmin/config.json``. Even if you have a single-node installation, ``ssh username@localhost`` needs to work properly. 8 | 9 | There are two ways to configure SSH: with keys so that you can use passwordless SSH, or with passwords. If your cluster already has passwordless SSH configured for the username ``user``, you can skip this step if the username is root, otherwise the root public key (id_rsa.pub) needs to be appended to the non-root username’s authorized_keys file. If you are intending to use ``presto-admin`` with passwords, take a look at the documentation below, because there are several ways to specify the password. 10 | 11 | Using ``presto-admin`` with passwordless SSH 12 | -------------------------------------------- 13 | In order to set up passwordless SSH, you must first login as username on the presto-admin node and generate keys with no passphrase on the node running ``presto-admin``: 14 | :: 15 | 16 | ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa 17 | 18 | While logged in as username, copy the public key to all of the coordinator and worker nodes: 19 | :: 20 | 21 | ssh @ "mkdir -p ~/.ssh && chmod 700 ~/.ssh" 22 | scp ~/.ssh/id_rsa.pub @:~/.ssh/id_rsa.pub 23 | 24 | Log into all of those nodes and append the public key to the authorized key file: 25 | :: 26 | 27 | ssh @ "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys" 28 | 29 | For non-root username, log into all of those nodes and append the root user public key to the username authorized key file, provided the passwordless ssh has been setup for root user.: 30 | :: 31 | 32 | ssh @ "sudo cat /root/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys" 33 | 34 | Once you have passwordless SSH set up, you can just run ``presto-admin`` commands as they appear in the documentation. If your private key is not in ``~/.ssh``, it is possible to specify one or several private keys using the -i CLI option: 35 | 36 | :: 37 | 38 | ./presto-admin -i -i 39 | 40 | 41 | Please also note that it is not common for servers to allow passwordless SSH for root because of security concerns, so it is preferable for the SSH user not to be root. 42 | 43 | Using ``presto-admin`` with SSH passwords 44 | ----------------------------------------- 45 | If you do not want to set up passwordless SSH on your cluster, it is possible to use ``presto-admin`` with SSH passwords. However, you will need to add a password argument to the ``presto-admin`` commands as they appear in the documentation. There are several options. To specify a password on the CLI in plaintext: 46 | 47 | :: 48 | 49 | ./presto-admin -p 50 | 51 | However, from a security perspective, it is preferable not to type your password in plaintext. Thus, it is also possible to add an interactive password prompt, which prompts you for the initial value of your password before running any commands: 52 | 53 | :: 54 | 55 | ./presto-admin -I 56 | Initial value for env.password: 57 | 58 | If you do not specify a password, the command will fail with a parallel execution failure, since, by default, ``presto-admin`` runs in parallel and cannot prompt for a password while running in parallel. If you specify the ``--serial`` option for ``presto-admin``, ``presto-admin`` will prompt you for a password if it cannot connect. 59 | 60 | Please note that the SSH password for the user specified in ``~/.prestoadmin/config.json`` must match the sudo password for that user. 61 | 62 | -------------------------------------------------------------------------------- /docs/user-guide.rst: -------------------------------------------------------------------------------- 1 | .. _comprehensive-guide-label: 2 | 3 | ********** 4 | User Guide 5 | ********** 6 | 7 | A full explanation of the commands and features of ``presto-admin``. 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | quick-start-guide 13 | emr 14 | installation/advanced-installation-options 15 | installation/presto-port-configuration 16 | ssh-configuration 17 | presto-admin-commands 18 | presto-admin-cli-options 19 | 20 | -------------------------------------------------------------------------------- /packaging/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | import os 19 | 20 | package_dir = os.path.abspath(os.path.dirname(__file__)) 21 | -------------------------------------------------------------------------------- /packaging/install-prestoadmin.template: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | set -e 17 | 18 | PYTHON_BIN=python 19 | 20 | while getopts ":p:" c; do 21 | case $c in 22 | p) 23 | PYTHON_BIN="$OPTARG" 24 | ;; 25 | \?) 26 | echo "Unrecognized option -$OPTARG" >&2 27 | exit 1 28 | ;; 29 | :) 30 | echo "Option -$OPTARG requires an argument" >&2 31 | exit 1 32 | ;; 33 | esac 34 | done 35 | 36 | if [ -d "third-party" ]; then 37 | tar xvzf third-party/virtualenv-%VIRTUALENV_VERSION%.tar.gz -C third-party || true 38 | "$PYTHON_BIN" third-party/virtualenv-%VIRTUALENV_VERSION%/virtualenv.py presto-admin-install 39 | else 40 | wget --no-check-certificate https://pypi.python.org/packages/source/v/virtualenv/virtualenv-%VIRTUALENV_VERSION%.tar.gz 41 | tar xvzf virtualenv-%VIRTUALENV_VERSION%.tar.gz || true 42 | "$PYTHON_BIN" virtualenv-%VIRTUALENV_VERSION%/virtualenv.py presto-admin-install 43 | fi 44 | 45 | source presto-admin-install/bin/activate 46 | cert_file=$1 47 | # trust pypi.python.org by default, otherwise use cert_file provided 48 | cert_options='--trusted-host pypi.python.org' 49 | if [ -n "$1" ]; then 50 | if [ ! -f $cert_file ]; then 51 | echo "Adding pypi.python.org as trusted-host. Cannot find certificate file: "$cert_file 52 | else 53 | cert_options='--cert '$cert_file 54 | fi 55 | fi 56 | 57 | pip install $cert_options %WHEEL_NAME%.whl %ONLINE_OR_OFFLINE_INSTALL% 58 | if ! `"$PYTHON_BIN" -c "import paramiko" > /dev/null 2>&1` ; then 59 | printf "\nERROR\n" 60 | echo "Paramiko could not be imported. This usually means that pycrypto (a dependency of paramiko)" 61 | echo "has been compiled against a different libc version. Ensure the presto-admin installer is " 62 | echo "built on the same OS as the target installation OS." 63 | exit 1 64 | fi 65 | deactivate 66 | 67 | cat > `pwd`/presto-admin << EOT 68 | #!/bin/bash 69 | export VIRTUAL_ENV="`pwd`/presto-admin-install" 70 | export PATH="\$VIRTUAL_ENV/bin:\$PATH" 71 | unset PYTHON_HOME 72 | 73 | exec presto-admin "\$@" 74 | EOT 75 | chmod 755 `pwd`/presto-admin 76 | 77 | CONF_DIR=${PRESTO_ADMIN_CONFIG_DIR:-~/.prestoadmin} 78 | mkdir -p "$CONF_DIR" 79 | 80 | LOG_DIR=${PRESTO_ADMIN_LOG_DIR:-$CONF_DIR/log} 81 | mkdir -p "$LOG_DIR" 82 | 83 | CATALOG_DIR=$CONF_DIR/catalog 84 | mkdir -p "$CATALOG_DIR" 85 | 86 | COORDINATOR_DIR=$CONF_DIR/coordinator 87 | mkdir -p "$COORDINATOR_DIR" 88 | 89 | WORKERS_DIR=$CONF_DIR/workers 90 | mkdir -p "$WORKERS_DIR" 91 | -------------------------------------------------------------------------------- /prestoadmin/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | """Presto-Admin tool for deploying and managing Presto clusters""" 17 | 18 | import os 19 | import sys 20 | import prestoadmin._version 21 | 22 | from fabric.api import env 23 | 24 | main_dir = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) 25 | 26 | import fabric_patches # noqa 27 | 28 | from prestoadmin.mode import get_mode, for_mode, MODE_STANDALONE, \ 29 | MODE_SLIDER # noqa 30 | from prestoadmin.util.exception import ConfigFileNotFoundError, \ 31 | ConfigurationError # noqa 32 | 33 | __version__ = prestoadmin._version.__version__ 34 | 35 | # 36 | # Subcommands common to all modes. If anybody knows why fabric_patches is in 37 | # the list, I'll make a note for the next person. 38 | # 39 | __all__ = ['fabric_patches'] 40 | 41 | cfg_mode = MODE_STANDALONE 42 | try: 43 | cfg_mode = get_mode() 44 | except ConfigFileNotFoundError as e: 45 | pass 46 | except ConfigurationError as e: 47 | print >>sys.stderr, e.message 48 | 49 | 50 | ADDITIONAL_TASK_MODULES = { 51 | MODE_SLIDER: [('yarn_slider.server', 'server'), 52 | ('yarn_slider.slider', 'slider')], 53 | MODE_STANDALONE: ['topology', 54 | ('configure_cmds', 'configuration'), 55 | 'server', 56 | 'catalog', 57 | 'package', 58 | 'collect', 59 | 'file', 60 | 'plugin']} 61 | 62 | 63 | if cfg_mode is not None: 64 | atms = for_mode(cfg_mode, ADDITIONAL_TASK_MODULES) 65 | for atm in atms: 66 | try: 67 | module, subcommand_name = atm 68 | except ValueError: 69 | module = atm 70 | subcommand_name = atm 71 | 72 | __all__.append(subcommand_name) 73 | 74 | components = module.split('.') 75 | 76 | if len(components) == 1: 77 | # The simple case... 78 | # import as 79 | globals()[subcommand_name] = __import__(module, globals()) 80 | else: 81 | # The complicated case: 82 | # import foo.bar doesn't actually import foo.bar; it imports foo. 83 | # This is why, for example, you can't to the following: 84 | 85 | # >>> import os.path 86 | # >>> path.join('foo', 'bar', 'baz', 'zot') 87 | # 88 | # Doing the equivalent of import yarn_slider.slider as slider 89 | # results in the global slider variable being assigned to the 90 | # yarn_slider module, which is NOT what we want. 91 | # Instead, we need to recursively traverse the submodules until we 92 | # get to the one we're interested in. 93 | submodule = __import__(module, globals()) 94 | for c in components[1:]: 95 | submodule = submodule.__dict__[c] 96 | globals()[subcommand_name] = submodule 97 | 98 | 99 | env.roledefs = { 100 | 'coordinator': [], 101 | 'worker': [], 102 | 'all': [] 103 | } 104 | -------------------------------------------------------------------------------- /prestoadmin/_version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Version information""" 15 | 16 | # This must be the last line in the file and the format must be maintained 17 | # even when the version is changed 18 | __version__ = '2.2-SNAPSHOT' 19 | -------------------------------------------------------------------------------- /prestoadmin/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Module for reading, writing, and processing configuration files 16 | """ 17 | import json 18 | import os 19 | import logging 20 | import errno 21 | import re 22 | 23 | from prestoadmin.util.exception import ConfigurationError,\ 24 | ConfigFileNotFoundError 25 | 26 | COMMENT_CHARS = ['!', '#'] 27 | _LOGGER = logging.getLogger(__name__) 28 | 29 | 30 | def get_conf_from_json_file(path): 31 | try: 32 | with open(path, 'r') as conf_file: 33 | if os.path.getsize(conf_file.name) == 0: 34 | return {} 35 | return json.load(conf_file) 36 | except IOError: 37 | raise ConfigFileNotFoundError( 38 | config_path=path, message="Missing configuration file %s." % 39 | (repr(path))) 40 | except ValueError as e: 41 | raise ConfigurationError(e) 42 | 43 | 44 | def get_conf_from_properties_file(path): 45 | with open(path, 'r') as conf_file: 46 | return get_conf_from_properties_data(conf_file) 47 | 48 | 49 | def get_conf_from_properties_data(data): 50 | props = {} 51 | for line in data.read().splitlines(): 52 | line = line.strip() 53 | if len(line) > 0 and line[0] not in COMMENT_CHARS: 54 | pair = split_to_pair(line) 55 | props[pair[0]] = pair[1] 56 | return props 57 | 58 | 59 | def split_to_pair(line): 60 | split_line = re.split(r'\s*(?=, " 65 | ": or ") 66 | return tuple(split_line) 67 | 68 | 69 | def get_conf_from_config_file(path): 70 | with open(path, 'r') as conf_file: 71 | settings = conf_file.read().splitlines() 72 | return settings 73 | 74 | 75 | def json_to_string(conf): 76 | return json.dumps(conf, indent=4, separators=(',', ':')) 77 | 78 | 79 | def write_conf_to_file(conf, path): 80 | # Note: this function expects conf to be flat 81 | # either a dict for .properties file or a list for .config 82 | ext = os.path.splitext(path)[1] 83 | if ext == ".properties": 84 | write_properties_file(conf, path) 85 | elif ext == ".config": 86 | write_config_file(conf, path) 87 | 88 | 89 | def write_properties_file(conf, path): 90 | output = '' 91 | for key, value in conf.iteritems(): 92 | output += '%s=%s\n' % (key, value) 93 | write(output, path) 94 | 95 | 96 | def write_config_file(conf, path): 97 | output = '\n'.join(conf) 98 | write(output, path) 99 | 100 | 101 | def write(output, path): 102 | conf_directory = os.path.dirname(path) 103 | try: 104 | os.makedirs(conf_directory) 105 | except OSError as e: 106 | if e.errno == errno.EEXIST: 107 | pass 108 | else: 109 | raise 110 | 111 | with open(path, 'w') as f: 112 | f.write(output) 113 | 114 | 115 | def fill_defaults(conf, defaults): 116 | try: 117 | default_items = defaults.iteritems() 118 | except AttributeError: 119 | return 120 | 121 | for k, v in default_items: 122 | conf.setdefault(k, v) 123 | fill_defaults(conf[k], v) 124 | -------------------------------------------------------------------------------- /prestoadmin/file.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Commands for running scripts on a cluster 17 | """ 18 | import logging 19 | from fabric.operations import put, sudo 20 | from fabric.decorators import task 21 | from fabric.api import env 22 | from os import path 23 | 24 | from prestoadmin.standalone.config import StandaloneConfig 25 | from prestoadmin.util.base_config import requires_config 26 | from prestoadmin.util.constants import REMOTE_COPY_DIR 27 | from prestoadmin.plugin import write 28 | 29 | _LOGGER = logging.getLogger(__name__) 30 | __all__ = ['run', 'copy'] 31 | 32 | 33 | @task 34 | @requires_config(StandaloneConfig) 35 | def run(script, remote_dir='/tmp'): 36 | """ 37 | Run an arbitrary script on all nodes in the cluster. 38 | 39 | Parameters: 40 | script - The path to the script 41 | remote_dir - Where to put the script on the cluster. Default is /tmp. 42 | """ 43 | script_name = path.basename(script) 44 | remote_path = path.join(remote_dir, script_name) 45 | put(script, remote_path) 46 | sudo('chmod u+x %s' % remote_path) 47 | sudo(remote_path) 48 | sudo('rm %s' % remote_path) 49 | 50 | 51 | @task 52 | @requires_config(StandaloneConfig) 53 | def copy(local_file, remote_dir=REMOTE_COPY_DIR): 54 | """ 55 | Copy a file to all nodes in the cluster. 56 | 57 | Parameters: 58 | local_file - The path to the file 59 | remote_dir - Where to put the file on the cluster. Default is /tmp. 60 | """ 61 | _LOGGER.info('copying file to %s' % env.host) 62 | write(local_file, remote_dir) 63 | -------------------------------------------------------------------------------- /prestoadmin/mode.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module for handling presto-admin mode-related functionality. 17 | """ 18 | 19 | import os 20 | 21 | from fabric.api import abort, task 22 | from fabric.decorators import runs_once 23 | 24 | from prestoadmin import config 25 | from prestoadmin.util.exception import ConfigurationError, \ 26 | ConfigFileNotFoundError 27 | from prestoadmin.util.local_config_util import get_config_directory 28 | 29 | MODE_CONF_PATH = os.path.join(get_config_directory(), 'mode.json') 30 | MODE_KEY = 'mode' 31 | 32 | MODE_SLIDER = 'yarn_slider' 33 | MODE_STANDALONE = 'standalone' 34 | 35 | VALID_MODES = [MODE_SLIDER, MODE_STANDALONE] 36 | 37 | 38 | def _load_mode_config(): 39 | return config.get_conf_from_json_file(MODE_CONF_PATH) 40 | 41 | 42 | def _store_mode_config(mode_config): 43 | config.write(config.json_to_string(mode_config), MODE_CONF_PATH) 44 | 45 | 46 | def get_mode(validate=True): 47 | mode_config = _load_mode_config() 48 | mode = mode_config.get(MODE_KEY) 49 | 50 | if validate and mode is None: 51 | raise ConfigurationError( 52 | 'Required key %s not found in configuration file %s' % ( 53 | MODE_KEY, MODE_CONF_PATH)) 54 | 55 | if validate and not validate_mode(mode): 56 | raise ConfigurationError( 57 | 'Invalid mode %s in configuration file %s. Valid modes are %s' % ( 58 | mode, MODE_CONF_PATH, ' '.join(VALID_MODES))) 59 | 60 | return mode 61 | 62 | 63 | def validate_mode(mode): 64 | return mode in VALID_MODES 65 | 66 | 67 | def for_mode(mode, mode_map): 68 | if sorted(mode_map.keys()) != sorted(VALID_MODES): 69 | raise Exception( 70 | 'keys in for_nodes\n%s\ndo not match VALID_MODES\n%s' % ( 71 | mode_map.keys(), VALID_MODES)) 72 | return mode_map[mode] 73 | 74 | 75 | @task 76 | @runs_once 77 | def select(new_mode): 78 | """ 79 | Change the mode. 80 | """ 81 | if not validate_mode(new_mode): 82 | abort('Invalid mode selection %s. Valid modes are %s' % ( 83 | new_mode, ' '.join(VALID_MODES))) 84 | 85 | mode_config = {} 86 | try: 87 | mode_config = _load_mode_config() 88 | except ConfigFileNotFoundError: 89 | pass 90 | 91 | mode_config[MODE_KEY] = new_mode 92 | _store_mode_config(mode_config) 93 | 94 | 95 | @task 96 | @runs_once 97 | def get(): 98 | """ 99 | Display the current mode. 100 | """ 101 | mode = None 102 | try: 103 | mode = get_mode(validate=False) 104 | print mode 105 | except ConfigFileNotFoundError: 106 | abort("Select a mode using the subcommand 'mode select '") 107 | 108 | 109 | @task 110 | @runs_once 111 | def list(): 112 | """ 113 | List the supported modes. 114 | """ 115 | print ' '.join(VALID_MODES) 116 | -------------------------------------------------------------------------------- /prestoadmin/node.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module for the presto coordinator's configuration. 17 | Loads and validates the coordinator.json file and creates the files needed 18 | to deploy on the presto cluster 19 | """ 20 | from abc import abstractmethod, ABCMeta 21 | import logging 22 | import os 23 | 24 | import config 25 | import presto_conf 26 | from prestoadmin.presto_conf import get_presto_conf 27 | 28 | _LOGGER = logging.getLogger(__name__) 29 | 30 | 31 | class Node(): 32 | __metaclass__ = ABCMeta 33 | 34 | def __init__(self): 35 | pass 36 | 37 | def get_conf(self): 38 | conf = get_presto_conf(self._get_conf_dir()) 39 | for name in presto_conf.REQUIRED_FILES: 40 | if name not in conf: 41 | _LOGGER.debug('%s configuration for %s not found. ' 42 | 'Default configuration will be deployed', 43 | type(self).__name__, name) 44 | conf_value = self.default_config(name) 45 | conf[name] = conf_value 46 | file_path = os.path.join(self._get_conf_dir(), name) 47 | config.write_conf_to_file(conf_value, file_path) 48 | 49 | self.validate(conf) 50 | return conf 51 | 52 | @abstractmethod 53 | def _get_conf_dir(self): 54 | pass 55 | 56 | @abstractmethod 57 | def default_config(self, filename): 58 | pass 59 | 60 | @staticmethod 61 | @abstractmethod 62 | def validate(conf): 63 | pass 64 | 65 | def build_all_defaults(self): 66 | conf = {} 67 | for name in presto_conf.REQUIRED_FILES: 68 | conf[name] = self.default_config(name) 69 | return conf 70 | -------------------------------------------------------------------------------- /prestoadmin/plugin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | module for tasks relating to presto plugins 17 | """ 18 | import logging 19 | from fabric.decorators import task 20 | from fabric.operations import sudo, put 21 | import os 22 | from fabric.api import env 23 | from prestoadmin.standalone.config import StandaloneConfig 24 | from prestoadmin.util.base_config import requires_config 25 | from prestoadmin.util.constants import REMOTE_PLUGIN_DIR 26 | 27 | __all__ = ['add_jar'] 28 | _LOGGER = logging.getLogger(__name__) 29 | 30 | 31 | def write(local_path, remote_dir): 32 | sudo("mkdir -p " + remote_dir) 33 | put(local_path, remote_dir, use_sudo=True) 34 | 35 | 36 | @task 37 | @requires_config(StandaloneConfig) 38 | def add_jar(local_path, plugin_name, plugin_dir=REMOTE_PLUGIN_DIR): 39 | """ 40 | Deploy jar for the specified plugin to the plugin directory. 41 | 42 | Parameters: 43 | local_path - Local path to the jar to be deployed 44 | plugin_name - Name of the plugin subdirectory to deploy jars to 45 | plugin_dir - (Optional) The plugin directory. If no directory is 46 | given, '/usr/lib/presto/lib/plugin' is used by default. 47 | """ 48 | _LOGGER.info('deploying jars on %s' % env.host) 49 | write(local_path, os.path.join(plugin_dir, plugin_name)) 50 | -------------------------------------------------------------------------------- /prestoadmin/presto-admin-logging.ini: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [logger_root] 5 | level=DEBUG 6 | handlers=file 7 | 8 | [handlers] 9 | keys=file 10 | 11 | [handler_file] 12 | class=prestoadmin.util.all_write_handler.AllWriteTimedRotatingFileHandler 13 | formatter=verbose 14 | args=('%(log_file_path)s', 'D', 7) 15 | 16 | [formatters] 17 | keys=verbose 18 | 19 | [formatter_verbose] 20 | format=%(asctime)s|%(process)d|%(thread)d|%(name)s|%(levelname)s|%(message)s 21 | -------------------------------------------------------------------------------- /prestoadmin/presto_conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Module for processing presto configuration files 16 | """ 17 | import logging 18 | import os 19 | 20 | from prestoadmin.config import get_conf_from_properties_file, \ 21 | get_conf_from_config_file 22 | from prestoadmin.util.exception import ConfigurationError 23 | 24 | 25 | REQUIRED_FILES = ["node.properties", "jvm.config", "config.properties"] 26 | PRESTO_FILES = ["node.properties", "jvm.config", "config.properties", 27 | "log.properties"] 28 | _LOGGER = logging.getLogger(__name__) 29 | 30 | 31 | def get_presto_conf(conf_dir): 32 | if os.path.isdir(conf_dir): 33 | file_list = [name for name in os.listdir(conf_dir) if 34 | name in PRESTO_FILES] 35 | else: 36 | _LOGGER.debug("No directory " + conf_dir) 37 | file_list = [] 38 | 39 | conf = {} 40 | for filename in file_list: 41 | ext = os.path.splitext(filename)[1] 42 | file_path = os.path.join(conf_dir, filename) 43 | if ext == ".properties": 44 | conf[filename] = get_conf_from_properties_file(file_path) 45 | elif ext == ".config": 46 | conf[filename] = get_conf_from_config_file(file_path) 47 | return conf 48 | 49 | 50 | def validate_presto_conf(conf): 51 | for required in REQUIRED_FILES: 52 | if required not in conf: 53 | raise ConfigurationError("Missing configuration for required " 54 | "file: " + required) 55 | 56 | expect_object_msg = "%s must be an object with key-value property pairs" 57 | if not isinstance(conf["node.properties"], dict): 58 | raise ConfigurationError(expect_object_msg % "node.properties") 59 | 60 | if not isinstance(conf["jvm.config"], list): 61 | raise ConfigurationError("jvm.config must contain a json array of jvm " 62 | "arguments ([arg1, arg2, arg3])") 63 | 64 | if not isinstance(conf["config.properties"], dict): 65 | raise ConfigurationError(expect_object_msg % "config.properties") 66 | 67 | return conf 68 | -------------------------------------------------------------------------------- /prestoadmin/standalone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/prestoadmin/standalone/__init__.py -------------------------------------------------------------------------------- /prestoadmin/topology.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module for setting and validating the presto-admin config 17 | """ 18 | import pprint 19 | 20 | from fabric.api import env, runs_once, task 21 | 22 | from prestoadmin.standalone.config import StandaloneConfig 23 | from prestoadmin.util.base_config import requires_config 24 | 25 | import prestoadmin.util.fabricapi as util 26 | 27 | 28 | @task 29 | @runs_once 30 | @requires_config(StandaloneConfig) 31 | def show(): 32 | """ 33 | Shows the current topology configuration for the cluster (including the 34 | coordinators, workers, SSH port, and SSH username) 35 | """ 36 | pprint.pprint(get_conf_from_fabric(), width=1) 37 | 38 | 39 | def get_conf_from_fabric(): 40 | return {'coordinator': util.get_coordinator_role()[0], 41 | 'workers': util.get_worker_role(), 42 | 'port': env.port, 43 | 'username': env.user} 44 | -------------------------------------------------------------------------------- /prestoadmin/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/prestoadmin/util/__init__.py -------------------------------------------------------------------------------- /prestoadmin/util/all_write_handler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from logging import handlers 16 | import os 17 | 18 | 19 | class AllWriteTimedRotatingFileHandler(handlers.TimedRotatingFileHandler): 20 | def _open(self): 21 | prev_umask = os.umask(000) 22 | rotating_file_handler = handlers.TimedRotatingFileHandler._open(self) 23 | os.umask(prev_umask) 24 | return rotating_file_handler 25 | -------------------------------------------------------------------------------- /prestoadmin/util/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | This modules contains read-only constants used throughout 17 | the presto admin project. 18 | """ 19 | 20 | import os 21 | 22 | import prestoadmin 23 | 24 | # Logging Config File Locations 25 | LOGGING_CONFIG_FILE_NAME = 'presto-admin-logging.ini' 26 | LOGGING_CONFIG_FILE_DIRECTORIES = [ 27 | os.path.join(prestoadmin.main_dir, 'prestoadmin') 28 | ] 29 | 30 | # local configuration 31 | LOG_DIR_ENV_VARIABLE = 'PRESTO_ADMIN_LOG_DIR' 32 | CONFIG_DIR_ENV_VARIABLE = 'PRESTO_ADMIN_CONFIG_DIR' 33 | LOCAL_CONF_DIR = '.prestoadmin' 34 | DEFAULT_LOCAL_CONF_DIR = os.path.join(os.path.expanduser('~'), LOCAL_CONF_DIR) 35 | TOPOLOGY_CONFIG_FILE = 'config.json' 36 | COORDINATOR_DIR_NAME = 'coordinator' 37 | WORKERS_DIR_NAME = 'workers' 38 | CATALOG_DIR_NAME = 'catalog' 39 | 40 | # remote configuration 41 | REMOTE_CONF_DIR = '/etc/presto' 42 | REMOTE_CATALOG_DIR = os.path.join(REMOTE_CONF_DIR, 'catalog') 43 | REMOTE_PACKAGES_PATH = '/opt/prestoadmin/packages' 44 | DEFAULT_PRESTO_SERVER_LOG_FILE = '/var/log/presto/server.log' 45 | DEFAULT_PRESTO_LAUNCHER_LOG_FILE = '/var/log/presto/launcher.log' 46 | REMOTE_PLUGIN_DIR = '/usr/lib/presto/lib/plugin' 47 | REMOTE_COPY_DIR = '/tmp' 48 | 49 | # Presto configuration files 50 | CONFIG_PROPERTIES = "config.properties" 51 | LOG_PROPERTIES = "log.properties" 52 | JVM_CONFIG = "jvm.config" 53 | NODE_PROPERTIES = "node.properties" 54 | -------------------------------------------------------------------------------- /prestoadmin/util/exception.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | This module defines error types relevant to the Presto 17 | administrative suite. 18 | """ 19 | import re 20 | 21 | import sys 22 | import traceback 23 | 24 | 25 | # Beware the nuances of pickling Exceptions: 26 | # http://bugs.python.org/issue1692335 27 | class ExceptionWithCause(Exception): 28 | 29 | def __init__(self, message=''): 30 | self.inner_exception = None 31 | 32 | causing_exception = sys.exc_info()[1] 33 | if causing_exception: 34 | self.inner_exception = traceback.format_exc() + \ 35 | ExceptionWithCause.get_cause_if_supported(causing_exception) 36 | 37 | super(ExceptionWithCause, self).__init__(message) 38 | 39 | @staticmethod 40 | def get_cause_if_supported(exception): 41 | try: 42 | inner = exception.inner_exception 43 | except AttributeError: 44 | inner = None 45 | 46 | if inner: 47 | return '\nCaused by:\n{tb}'.format( 48 | tb=inner 49 | ) 50 | else: 51 | return '' 52 | 53 | 54 | class InvalidArgumentError(ExceptionWithCause): 55 | pass 56 | 57 | 58 | class ConfigurationError(ExceptionWithCause): 59 | pass 60 | 61 | 62 | class ConfigFileNotFoundError(ConfigurationError): 63 | def __init__(self, message='', config_path=''): 64 | super(ConfigFileNotFoundError, self).__init__(message) 65 | self.config_path = config_path 66 | 67 | 68 | def is_arguments_error(exception): 69 | return isinstance(exception, TypeError) and \ 70 | re.match(r'.+\(\) takes (at most \d+|no|exactly \d+|at least \d+) ' 71 | r'arguments? \(\d+ given\)', exception.message) 72 | -------------------------------------------------------------------------------- /prestoadmin/util/fabric_application.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Logic for starting and stopping Fabric applications. 17 | """ 18 | 19 | from fabric.network import disconnect_all 20 | from prestoadmin.util.application import Application 21 | 22 | import logging 23 | import sys 24 | 25 | 26 | # Normally this would use the logger for __name__, however, this is 27 | # effectively the "root" logger for the application. If this code 28 | # were running directly in the executable script __name__ would be 29 | # set to '__main__', so we emulate that same behavior here. This should 30 | # resolve to the same logger that will be used by the entry point script. 31 | logger = logging.getLogger('__main__') 32 | 33 | 34 | class FabricApplication(Application): 35 | """ 36 | A Presto Fabric application entry point. Provides logging and exception 37 | handling features. Additionally cleans up Fabric network connections 38 | before exiting. 39 | """ 40 | 41 | def _exit_cleanup_hook(self): 42 | """ 43 | Disconnect all Fabric connections in addition to shutting down the 44 | logging. 45 | """ 46 | disconnect_all() 47 | Application._exit_cleanup_hook(self) 48 | 49 | def _handle_error(self): 50 | """ 51 | Handle KeyboardInterrupt in a special way: don't indicate 52 | that it's an error. 53 | 54 | Returns: 55 | Nothing 56 | """ 57 | self._log_exception() 58 | if isinstance(self.exception, KeyboardInterrupt): 59 | print >> sys.stderr, "Stopped." 60 | sys.exit(0) 61 | else: 62 | Application._handle_error(self) 63 | -------------------------------------------------------------------------------- /prestoadmin/util/fabricapi.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module to add extensions and helpers for fabric api methods 17 | """ 18 | 19 | from functools import wraps 20 | 21 | from fabric.api import env, put, settings, sudo 22 | from fabric.utils import abort 23 | 24 | 25 | def get_host_list(): 26 | return [host for host in env.hosts if host not in env.exclude_hosts] 27 | 28 | 29 | def get_coordinator_role(): 30 | return env.roledefs['coordinator'] 31 | 32 | 33 | def get_worker_role(): 34 | return env.roledefs['worker'] 35 | 36 | 37 | def task_by_rolename(rolename): 38 | def inner_decorator(f): 39 | @wraps(f) 40 | def wrapper(*args, **kwargs): 41 | return by_rolename(env.host, rolename, f, *args, **kwargs) 42 | return wrapper 43 | return inner_decorator 44 | 45 | 46 | def by_rolename(host, rolename, f, *args, **kwargs): 47 | if rolename is None: 48 | f(*args, **kwargs) 49 | else: 50 | if rolename not in env.roledefs.keys(): 51 | abort("Invalid role name %s. Valid rolenames are %s" % 52 | (rolename, env.roledefs.keys())) 53 | if host in env.roledefs[rolename]: 54 | return f(*args, **kwargs) 55 | 56 | 57 | def by_role_coordinator(host, f, *args, **kwargs): 58 | if host in get_coordinator_role(): 59 | return f(*args, **kwargs) 60 | 61 | 62 | def by_role_worker(host, f, *args, **kwargs): 63 | if host in get_worker_role() and host not in get_coordinator_role(): 64 | return f(*args, **kwargs) 65 | 66 | 67 | def put_secure(user_group, mode, *args, **kwargs): 68 | missing_owner_code = 42 69 | user, group = user_group.split(":") 70 | 71 | files = put(*args, mode=mode, **kwargs) 72 | 73 | for file in files: 74 | with settings(warn_only=True): 75 | command = \ 76 | "( getent passwd {user} >/dev/null || ( rm -f {file} ; " \ 77 | "exit {missing_owner_code} ) ) && " \ 78 | "chown {user_group} {file}".format( 79 | user=user, file=file, user_group=user_group, 80 | missing_owner_code=missing_owner_code) 81 | 82 | result = sudo(command) 83 | 84 | if result.return_code == missing_owner_code: 85 | abort("User %s does not exist. Make sure the Presto " 86 | "server RPM is installed and try again" % (user,)) 87 | elif result.failed: 88 | abort("Failed to chown file %s" % (file,)) 89 | -------------------------------------------------------------------------------- /prestoadmin/util/filesystem.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Filesystem tools.""" 16 | 17 | import errno 18 | import logging 19 | import os 20 | 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | def ensure_parent_directories_exist(path): 26 | try: 27 | os.makedirs(os.path.dirname(path)) 28 | except OSError as e: 29 | if e.errno != errno.EEXIST: 30 | raise e 31 | 32 | 33 | def ensure_directory_exists(path): 34 | try: 35 | os.makedirs(path) 36 | except OSError as e: 37 | if e.errno != errno.EEXIST: 38 | raise e 39 | 40 | 41 | def write_to_file_if_not_exists(content, path): 42 | flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY 43 | 44 | try: 45 | os.makedirs(os.path.dirname(path)) 46 | except OSError as e: 47 | if e.errno == errno.EEXIST: 48 | pass 49 | else: 50 | raise 51 | 52 | try: 53 | file_handle = os.open(path, flags) 54 | except OSError as e: 55 | if e.errno == errno.EEXIST: 56 | pass 57 | else: 58 | raise 59 | else: 60 | with os.fdopen(file_handle, 'w') as f: 61 | f.write(content) 62 | -------------------------------------------------------------------------------- /prestoadmin/util/hiddenoptgroup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | An option group for which you can hide the help text. 17 | """ 18 | 19 | import logging 20 | from optparse import OptionGroup 21 | 22 | 23 | _LOGGER = logging.getLogger(__name__) 24 | 25 | 26 | class HiddenOptionGroup(OptionGroup): 27 | """ 28 | Optparse allows you to suppress Options from the help text, but not 29 | groups. This class allows you to suppress the help of groups. 30 | """ 31 | 32 | def __init__(self, parser, title, description=None, suppress_help=False): 33 | OptionGroup.__init__(self, parser, title, description) 34 | self.suppress_help = suppress_help 35 | 36 | def format_help(self, formatter): 37 | if not self.suppress_help: 38 | return OptionGroup.format_help(self, formatter) 39 | else: 40 | return "" 41 | -------------------------------------------------------------------------------- /prestoadmin/util/httpscacertconnection.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import ssl 3 | import httplib 4 | 5 | # Adapted from http://code.activestate.com/recipes/577548-https-httplib-client-connection-with-certificate-v/ 6 | # BSD-licensed. 7 | 8 | 9 | class HTTPSCaCertConnection(httplib.HTTPSConnection): 10 | """ Class to make a HTTPS connection, with support for full client-based SSL Authentication""" 11 | 12 | def __init__(self, host, port, key_file, cert_file, ca_file, strict, timeout=None): 13 | httplib.HTTPSConnection.__init__(self, host, port, key_file, cert_file, strict, timeout) 14 | self.key_file = key_file 15 | self.cert_file = cert_file 16 | self.ca_file = ca_file 17 | self.timeout = timeout 18 | 19 | def connect(self): 20 | """ Connect to a host on a given (SSL) port. 21 | If ca_file is pointing somewhere, use it to check Server Certificate. 22 | 23 | Redefined/copied and extended from httplib.py:1105 (Python 2.6.x). 24 | This is needed to pass cert_reqs=ssl.CERT_REQUIRED as parameter to ssl.wrap_socket(), 25 | which forces SSL to check server certificate against our client certificate. 26 | """ 27 | sock = socket.create_connection((self.host, self.port), self.timeout) 28 | if self._tunnel_host: 29 | self.sock = sock 30 | self._tunnel() 31 | # If there's no CA File, don't force Server Certificate Check 32 | if self.ca_file: 33 | self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ca_certs=self.ca_file, 34 | cert_reqs=ssl.CERT_REQUIRED) 35 | else: 36 | self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, cert_reqs=ssl.CERT_NONE) 37 | -------------------------------------------------------------------------------- /prestoadmin/util/local_config_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | 16 | from prestoadmin.util.constants import LOG_DIR_ENV_VARIABLE, CONFIG_DIR_ENV_VARIABLE, DEFAULT_LOCAL_CONF_DIR, \ 17 | TOPOLOGY_CONFIG_FILE, COORDINATOR_DIR_NAME, WORKERS_DIR_NAME, CATALOG_DIR_NAME 18 | 19 | 20 | def get_config_directory(): 21 | config_directory = os.environ.get(CONFIG_DIR_ENV_VARIABLE) 22 | if not config_directory: 23 | config_directory = DEFAULT_LOCAL_CONF_DIR 24 | return config_directory 25 | 26 | 27 | def get_log_directory(): 28 | config_directory = os.environ.get(LOG_DIR_ENV_VARIABLE) 29 | if not config_directory: 30 | config_directory = os.path.join(get_config_directory(), 'log') 31 | return config_directory 32 | 33 | 34 | def get_topology_path(): 35 | return os.path.join(get_config_directory(), TOPOLOGY_CONFIG_FILE) 36 | 37 | 38 | def get_coordinator_directory(): 39 | return os.path.join(get_config_directory(), COORDINATOR_DIR_NAME) 40 | 41 | 42 | def get_workers_directory(): 43 | return os.path.join(get_config_directory(), WORKERS_DIR_NAME) 44 | 45 | 46 | def get_catalog_directory(): 47 | return os.path.join(get_config_directory(), CATALOG_DIR_NAME) 48 | -------------------------------------------------------------------------------- /prestoadmin/util/parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | An extension to optparse for presto-admin which logs user parsing errors. 17 | """ 18 | 19 | import logging 20 | from optparse import OptionParser 21 | import sys 22 | 23 | 24 | _LOGGER = logging.getLogger(__name__) 25 | 26 | 27 | class LoggingOptionParser(OptionParser): 28 | """ 29 | An extension to optparse which logs exceptions via the logging 30 | module in addition to writing the out to stderr. 31 | 32 | If used with HiddenOptionGroup, print_extended_help disables the 33 | suppress_help attribute of HiddenOptionGroup so as to print out 34 | extended helptext. 35 | """ 36 | 37 | def exit(self, status=0, msg=None): 38 | _LOGGER.debug("Exiting option parser!") 39 | if msg: 40 | sys.stderr.write(msg) 41 | _LOGGER.error(msg) 42 | sys.exit(status) 43 | 44 | def print_extended_help(self, filename=None): 45 | old_suppress_help = {} 46 | for group in self.option_groups: 47 | try: 48 | old_suppress_help[group] = group.suppress_help 49 | group.suppress_help = False 50 | except AttributeError as e: 51 | old_suppress_help[group] = None 52 | _LOGGER.debug("Option group does not have option to " 53 | "suppress help; exception is " + e.message) 54 | self.print_help(file=filename) 55 | 56 | for group in self.option_groups: 57 | # Restore the suppressed help when applicable 58 | if old_suppress_help[group]: 59 | group.suppress_help = True 60 | 61 | def format_epilog(self, formatter): 62 | """ 63 | The default format_epilog strips the newlines (using textwrap), 64 | so we override format_epilog here to use its own epilog 65 | """ 66 | if not self.epilog: 67 | self.epilog = "" 68 | return self.epilog 69 | -------------------------------------------------------------------------------- /prestoadmin/util/remote_config_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import logging 16 | from fabric.context_managers import settings, hide 17 | from fabric.operations import sudo 18 | from fabric.tasks import execute 19 | from prestoadmin.util.exception import ConfigurationError 20 | from prestoadmin.util.constants import DEFAULT_PRESTO_LAUNCHER_LOG_FILE,\ 21 | DEFAULT_PRESTO_SERVER_LOG_FILE, REMOTE_CONF_DIR, REMOTE_CATALOG_DIR 22 | import prestoadmin.util.validators 23 | 24 | _LOGGER = logging.getLogger(__name__) 25 | 26 | NODE_CONFIG_FILE = REMOTE_CONF_DIR + '/node.properties' 27 | GENERAL_CONFIG_FILE = REMOTE_CONF_DIR + '/config.properties' 28 | 29 | 30 | def lookup_port(host): 31 | """ 32 | Get the http port from config.properties http-server.http.port property 33 | if available. 34 | If the property is missing return default port 8080. 35 | If the file is missing or cannot parse the port number, 36 | throw ConfigurationError 37 | :param host: 38 | :return: 39 | """ 40 | port = lookup_in_config('http-server.http.port', GENERAL_CONFIG_FILE, host) 41 | if not port: 42 | _LOGGER.info('Could not find property http-server.http.port.' 43 | 'Defaulting to 8080.') 44 | return 8080 45 | try: 46 | port = port.split('=', 1)[1] 47 | port = prestoadmin.util.validators.validate_port(port) 48 | _LOGGER.info('Looked up port ' + str(port) + ' on host ' + 49 | host) 50 | return port 51 | except ConfigurationError as e: 52 | raise ConfigurationError(e.message + 53 | ' for property ' 54 | 'http-server.http.port on host ' + 55 | host + '.') 56 | 57 | 58 | def lookup_server_log_file(host): 59 | try: 60 | return lookup_string_config('node.server-log-file', NODE_CONFIG_FILE, 61 | host, DEFAULT_PRESTO_SERVER_LOG_FILE) 62 | except: 63 | return DEFAULT_PRESTO_SERVER_LOG_FILE 64 | 65 | 66 | def lookup_launcher_log_file(host): 67 | try: 68 | return lookup_string_config('node.launcher-log-file', NODE_CONFIG_FILE, 69 | host, DEFAULT_PRESTO_LAUNCHER_LOG_FILE) 70 | except: 71 | return DEFAULT_PRESTO_LAUNCHER_LOG_FILE 72 | 73 | 74 | def lookup_catalog_directory(host): 75 | try: 76 | return lookup_string_config('catalog.config-dir', NODE_CONFIG_FILE, 77 | host, REMOTE_CATALOG_DIR) 78 | except: 79 | return REMOTE_CATALOG_DIR 80 | 81 | 82 | def lookup_string_config(config_value, config_file, host, default=''): 83 | value = lookup_in_config(config_value, config_file, host) 84 | if value: 85 | return value.split('=', 1)[1] 86 | else: 87 | return default 88 | 89 | 90 | def lookup_in_config(config_key, config_file, host): 91 | with settings(hide('stdout', 'warnings', 'aborts')): 92 | config_value = execute(sudo, 'grep %s= %s' % (config_key, config_file), 93 | user='presto', 94 | warn_only=True, host=host)[host] 95 | 96 | if isinstance(config_value, Exception) or config_value.return_code == 2: 97 | raise ConfigurationError('Could not access config file %s on ' 98 | 'host %s' % (config_file, host)) 99 | 100 | return config_value 101 | -------------------------------------------------------------------------------- /prestoadmin/util/validators.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module for validating configuration information supplied by the user. 17 | """ 18 | import re 19 | import socket 20 | 21 | from fabric.context_managers import settings 22 | from fabric.operations import run, sudo 23 | 24 | from prestoadmin.util.exception import ConfigurationError 25 | 26 | 27 | def validate_username(username): 28 | if not isinstance(username, basestring): 29 | raise ConfigurationError('Username must be of type string.') 30 | return username 31 | 32 | 33 | def validate_port(port): 34 | try: 35 | port_int = int(port) 36 | except TypeError: 37 | raise ConfigurationError('Port must be of type string, but ' 38 | 'found ' + str(type(port)) + '.') 39 | except ValueError: 40 | raise ConfigurationError('Invalid port number ' + port + 41 | ': port must be a number between 1 and 65535') 42 | if not port_int > 0 or not port_int < 65535: 43 | raise ConfigurationError('Invalid port number ' + port + 44 | ': port must be a number between 1 and 65535') 45 | return port_int 46 | 47 | 48 | def validate_host(host): 49 | try: 50 | socket.inet_pton(socket.AF_INET, host) 51 | return host 52 | except TypeError: 53 | raise ConfigurationError('Host must be of type string. Found ' + 54 | str(type(host)) + '.') 55 | except socket.error: 56 | pass 57 | 58 | try: 59 | socket.inet_pton(socket.AF_INET6, host) 60 | return host 61 | except socket.error: 62 | pass 63 | 64 | if not is_valid_hostname(host): 65 | raise ConfigurationError(repr(host) + ' is not a valid ' 66 | 'ip address or host name.') 67 | return host 68 | 69 | 70 | def is_valid_hostname(hostname): 71 | valid_name = '^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*' \ 72 | '([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' 73 | return re.match(valid_name, hostname) 74 | 75 | 76 | def validate_can_connect(user, host, port): 77 | with settings(host_string='%s@%s:%d' % (user, host, port), user=user): 78 | return run('exit 0').succeeded 79 | 80 | 81 | def validate_can_sudo(sudo_user, conn_user, host, port): 82 | with settings(host_string='%s@%s:%d' % (conn_user, host, port), 83 | warn_only=True): 84 | return sudo('exit 0', user=sudo_user).succeeded 85 | -------------------------------------------------------------------------------- /prestoadmin/yarn_slider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/prestoadmin/yarn_slider/__init__.py -------------------------------------------------------------------------------- /prestoadmin/yarn_slider/slider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module for installing and uninstalling slider. 17 | """ 18 | 19 | import os 20 | 21 | from fabric.api import env, task, abort 22 | from fabric.operations import put, sudo 23 | 24 | from prestoadmin.yarn_slider.config import SliderConfig, \ 25 | DIR, SLIDER_MASTER 26 | from prestoadmin.util.base_config import requires_config 27 | 28 | from prestoadmin.util.fabricapi import task_by_rolename 29 | 30 | __all__ = ['install', 'uninstall'] 31 | 32 | 33 | @task 34 | @requires_config(SliderConfig) 35 | @task_by_rolename(SLIDER_MASTER) 36 | def install(slider_tarball): 37 | """ 38 | Install slider on the slider master. You must provide a tar file on the 39 | local machine that contains the slider distribution. 40 | 41 | :param slider_tarball: The gzipped tar file containing the Apache Slider 42 | distribution 43 | """ 44 | deploy_install(slider_tarball) 45 | 46 | 47 | def deploy_install(slider_tarball): 48 | slider_dir = env.conf[DIR] 49 | slider_parent = os.path.dirname(slider_dir) 50 | slider_file = os.path.join(slider_parent, os.path.basename(slider_tarball)) 51 | 52 | sudo('mkdir -p %s' % (slider_dir)) 53 | 54 | result = put(slider_tarball, os.path.join(slider_parent, slider_file)) 55 | if result.failed: 56 | abort('Failed to send slider tarball %s to directory %s on host %s' % 57 | (slider_tarball, slider_dir, env.host)) 58 | 59 | sudo('gunzip -c %s | tar -x -C %s --strip-components=1 && rm -f %s' % 60 | (slider_file, slider_dir, slider_file)) 61 | 62 | 63 | @task 64 | @requires_config(SliderConfig) 65 | @task_by_rolename(SLIDER_MASTER) 66 | def uninstall(): 67 | """ 68 | Uninstall slider from the slider master. 69 | """ 70 | sudo('rm -r "%s"' % (env.conf[DIR])) 71 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | argparse==1.4 2 | paramiko==1.15.3 3 | flake8==2.5.4 4 | mock==1.0.1 5 | py==1.4.26 6 | Sphinx==1.3.1 7 | tox==1.9.2 8 | virtualenv==12.0.7 9 | wheel==0.23.0 10 | fabric==1.10.1 11 | requests==2.7.0 12 | docker-py==1.5.0 13 | certifi==2015.4.28 14 | nose==1.3.7 15 | nose-timer==0.6 16 | fudge==1.1.0 17 | PyYAML==3.11 18 | overrides==0.5 19 | setuptools==20.1.1 20 | pip==8.1.2 21 | retrying==1.3.3 22 | pyjks==0.5.1 23 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 0 3 | [nosetests] 4 | verbosity=3 5 | [flake8] 6 | max-line-length = 120 7 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/bare_image_provider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Abstract base class for bare image providers. 17 | 18 | Bare image providers know how to bring bare docker images into existence for 19 | the product tests. 20 | """ 21 | 22 | import abc 23 | 24 | from docker import Client 25 | 26 | 27 | class BareImageProvider(object): 28 | __metaclass__ = abc.ABCMeta 29 | 30 | def __init__(self, tag_decoration): 31 | super(BareImageProvider, self).__init__() 32 | self.tag_decoration = tag_decoration 33 | 34 | @abc.abstractmethod 35 | def create_bare_images(self, cluster, master_name, slave_name): 36 | """Create master and slave images to be tagged with master_name and 37 | slave_name, respectively.""" 38 | pass 39 | 40 | def get_tag_decoration(self): 41 | """Returns a string that's prepended to docker image tags for images 42 | based off of the bare image created by the provider.""" 43 | return self.tag_decoration 44 | 45 | 46 | """ 47 | Provides bare images from existing tags in Docker. For some of the heftier 48 | images, we don't want to go through a long and drawn-out Docker build on a 49 | regular basis. For these, we count on having an image in Docker that we can 50 | tag appropriately into the teradatalabs/pa_tests namespace. Test cleanup can 51 | continue to obliterate that namespace without disrupting the actual heavyweight 52 | images. 53 | 54 | As an additional benefit, this means we can have tests depend on images that 55 | the test code doesn't know how to build. That seems like a liability, but it 56 | that the build process for complex images can be versioned outside of the 57 | presto-admin codebase. 58 | """ 59 | 60 | 61 | class TagBareImageProvider(BareImageProvider): 62 | def __init__( 63 | self, base_master_name, base_slave_name, base_tag, tag_decoration): 64 | super(TagBareImageProvider, self).__init__(tag_decoration) 65 | self.base_master_name = base_master_name + ":" + base_tag 66 | self.base_slave_name = base_slave_name + ":" + base_tag 67 | self.client = Client() 68 | 69 | def create_bare_images(self, cluster, master_name, slave_name): 70 | self.client.tag(self.base_master_name, master_name) 71 | self.client.tag(self.base_slave_name, slave_name) 72 | -------------------------------------------------------------------------------- /tests/base_installer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Abstract base class for installers. 17 | """ 18 | 19 | import abc 20 | 21 | 22 | class BaseInstaller(object): 23 | __metaclass__ = abc.ABCMeta 24 | 25 | @staticmethod 26 | @abc.abstractmethod 27 | def get_dependencies(): 28 | """Returns a list of installers that need to be run prior to running 29 | this one. Dependencies are considered satisfied if their 30 | assert_installed() returns without asserting. 31 | """ 32 | raise NotImplementedError() 33 | 34 | @abc.abstractmethod 35 | def install(self): 36 | """Run the installer on the cluster. 37 | 38 | Installers may install something on one or more hosts of a cluster. 39 | After calling install(), the installer's assert_installed method should 40 | pass. 41 | """ 42 | pass 43 | 44 | @abc.abstractmethod 45 | def get_keywords(self, *args, **kwargs): 46 | """Get a map of keyword: value mappings. 47 | 48 | We do a bunch of string formatting in the product tests when comparing 49 | actual command output to expected output. Installers can use this 50 | method to return additional keywords to be used in string formatting. 51 | """ 52 | pass 53 | 54 | @staticmethod 55 | @abc.abstractmethod 56 | def assert_installed(testcase): 57 | """Check the cluster and assert if the installer hasn't been run. This 58 | should return without asserting if install() has been run. 59 | """ 60 | raise NotImplementedError() 61 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/integration/__init__.py -------------------------------------------------------------------------------- /tests/integration/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/integration/util/__init__.py -------------------------------------------------------------------------------- /tests/integration/util/data/presto-admin-logging.ini: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [logger_root] 5 | level=DEBUG 6 | handlers=file 7 | 8 | [handlers] 9 | keys=file 10 | 11 | [handler_file] 12 | class=handlers.TimedRotatingFileHandler 13 | formatter=verbose 14 | args=('%(log_file_path)s', 'D', 7) 15 | 16 | [formatters] 17 | keys=verbose 18 | 19 | [formatter_verbose] 20 | format=%(asctime)s|%(process)d|%(thread)d|%(name)s|%(levelname)s|%(message)s 21 | -------------------------------------------------------------------------------- /tests/integration/util/test_application.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import os 17 | import tempfile 18 | from unittest import TestCase 19 | 20 | from prestoadmin.util import constants 21 | from prestoadmin.util.application import Application 22 | from prestoadmin.util.constants import LOG_DIR_ENV_VARIABLE 23 | from prestoadmin.util.local_config_util import get_log_directory 24 | 25 | EXECUTABLE_NAME = 'foo.py' 26 | APPLICATION_NAME = 'foo' 27 | 28 | 29 | class ApplicationTest(TestCase): 30 | def setUp(self): 31 | # put log files in a temporary dir 32 | self.__old_prestoadmin_log = get_log_directory() 33 | self.__temporary_dir_path = tempfile.mkdtemp(prefix='app-int-test-') 34 | os.environ[LOG_DIR_ENV_VARIABLE] = self.__temporary_dir_path 35 | 36 | # monkey patch in a fake logging config file 37 | self.__old_log_dirs = list(constants.LOGGING_CONFIG_FILE_DIRECTORIES) 38 | constants.LOGGING_CONFIG_FILE_DIRECTORIES.append( 39 | os.path.join(os.path.dirname(__file__), 'data') 40 | ) 41 | 42 | # basicConfig is a noop if there are already handlers 43 | # present on the root logger, remove them all here 44 | self.__old_log_handlers = [] 45 | for handler in logging.root.handlers: 46 | self.__old_log_handlers.append(handler) 47 | logging.root.removeHandler(handler) 48 | 49 | def tearDown(self): 50 | constants.LOGGING_CONFIG_FILE_DIRECTORIES = self.__old_log_dirs 51 | 52 | # restore the log location 53 | if self.__old_prestoadmin_log: 54 | os.environ[LOG_DIR_ENV_VARIABLE] = self.__old_prestoadmin_log 55 | else: 56 | os.environ.pop(LOG_DIR_ENV_VARIABLE) 57 | 58 | # clean up the temporary directory 59 | os.system('rm -rf ' + self.__temporary_dir_path) 60 | 61 | # restore the old log handlers 62 | for handler in logging.root.handlers: 63 | logging.root.removeHandler(handler) 64 | for handler in self.__old_log_handlers: 65 | logging.root.addHandler(handler) 66 | 67 | def test_log_file_is_created(self): 68 | with Application(APPLICATION_NAME): 69 | pass 70 | 71 | log_file_path = os.path.join( 72 | get_log_directory(), 73 | APPLICATION_NAME + '.log' 74 | ) 75 | self.assertTrue( 76 | os.path.exists(log_file_path), 77 | 'Expected log file does not exist' 78 | ) 79 | self.assertTrue( 80 | os.path.getsize(log_file_path) > 0, 81 | 'Log file is empty' 82 | ) 83 | -------------------------------------------------------------------------------- /tests/no_hadoop_bare_image_provider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Provides bare images for standalone clusters. 17 | """ 18 | 19 | from tests.bare_image_provider import TagBareImageProvider 20 | 21 | 22 | class NoHadoopBareImageProvider(TagBareImageProvider): 23 | def __init__(self, images_tag): 24 | super(NoHadoopBareImageProvider, self).__init__( 25 | 'teradatalabs/centos6-ssh-oj8', 'teradatalabs/centos6-ssh-oj8', 26 | images_tag, 'nohadoop') 27 | -------------------------------------------------------------------------------- /tests/product/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import contextlib 16 | import os 17 | 18 | from exceptions import Exception 19 | 20 | 21 | def determine_jdk_directory(cluster): 22 | """ 23 | Return the directory where the JDK is installed. For example if the JDK is 24 | located in /usr/java/jdk1.8_91, then this method will return the string 25 | 'jdk1.8_91'. 26 | 27 | This method will throw an Exception if the number of JDKs matching the 28 | /usr/java/jdk* pattern is not equal to 1. 29 | 30 | :param cluster: cluster on which to search for the JDK directory 31 | """ 32 | number_of_jdks = cluster.exec_cmd_on_host(cluster.master, 'bash -c "ls -ld /usr/java/j*| wc -l"') 33 | if int(number_of_jdks) != 1: 34 | raise Exception('The number of JDK directories matching /usr/java/jdk* is not 1') 35 | output = cluster.exec_cmd_on_host(cluster.master, 'ls -d /usr/java/j*') 36 | return output.split(os.path.sep)[-1].strip('\n') 37 | 38 | 39 | @contextlib.contextmanager 40 | def relocate_jdk_directory(cluster, destination): 41 | """ 42 | Temporarily move the JDK to the destination directory 43 | 44 | :param cluster: cluster object on which to relocate the JDK directory 45 | :param destination: destination parent JDK directory, e.g. /tmp/ 46 | :returns the new full JDK directory, e.g. /tmp/jdk1.8_91 47 | """ 48 | # assume that Java is installed in the same folder on all nodes 49 | jdk_directory = determine_jdk_directory(cluster) 50 | source_jdk = os.path.join('/usr/java', jdk_directory) 51 | destination_jdk = os.path.join(destination, jdk_directory) 52 | for host in cluster.all_hosts(): 53 | cluster.exec_cmd_on_host( 54 | host, "mv %s %s" % (source_jdk, destination_jdk), invoke_sudo=True) 55 | 56 | yield destination_jdk 57 | 58 | for host in cluster.all_hosts(): 59 | cluster.exec_cmd_on_host( 60 | host, "mv %s %s" % (destination_jdk, source_jdk), invoke_sudo=True) 61 | -------------------------------------------------------------------------------- /tests/product/cluster_types.py: -------------------------------------------------------------------------------- 1 | from tests.product.mode_installers import StandaloneModeInstaller 2 | from tests.product.prestoadmin_installer import PrestoadminInstaller 3 | from tests.product.topology_installer import TopologyInstaller 4 | from tests.product.standalone.presto_installer import StandalonePrestoInstaller 5 | 6 | 7 | STANDALONE_BARE_CLUSTER = 'bare' 8 | BARE_CLUSTER = 'bare' 9 | STANDALONE_PA_CLUSTER = 'pa_only_standalone' 10 | STANDALONE_PRESTO_CLUSTER = 'presto' 11 | 12 | cluster_types = { 13 | BARE_CLUSTER: [], 14 | STANDALONE_PA_CLUSTER: [PrestoadminInstaller, 15 | StandaloneModeInstaller], 16 | STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller, 17 | StandaloneModeInstaller, 18 | TopologyInstaller, 19 | StandalonePrestoInstaller], 20 | } 21 | -------------------------------------------------------------------------------- /tests/product/config_dir_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from prestoadmin.util.constants import COORDINATOR_DIR_NAME, WORKERS_DIR_NAME, CATALOG_DIR_NAME 4 | 5 | 6 | # gets the information for presto-admin config directories on the cluster 7 | def get_config_directory(): 8 | return os.path.join('~', '.prestoadmin') 9 | 10 | 11 | def get_config_file_path(): 12 | return os.path.join(get_config_directory(), 'config.json') 13 | 14 | 15 | def get_coordinator_directory(): 16 | return os.path.join(get_config_directory(), COORDINATOR_DIR_NAME) 17 | 18 | 19 | def get_workers_directory(): 20 | return os.path.join(get_config_directory(), WORKERS_DIR_NAME) 21 | 22 | 23 | def get_catalog_directory(): 24 | return os.path.join(get_config_directory(), CATALOG_DIR_NAME) 25 | 26 | 27 | def get_log_directory(): 28 | return os.path.join(get_config_directory(), 'log') 29 | 30 | 31 | def get_mode_config_path(): 32 | return os.path.join(get_config_directory(), 'mode.json') 33 | 34 | 35 | def get_install_directory(): 36 | return os.path.join('~', 'prestoadmin') 37 | 38 | 39 | def get_presto_admin_path(): 40 | return os.path.join(get_install_directory(), 'presto-admin') 41 | -------------------------------------------------------------------------------- /tests/product/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module defining constants global to the product tests 17 | """ 18 | 19 | import json 20 | import os 21 | import sys 22 | 23 | import prestoadmin 24 | from prestoadmin import main_dir 25 | 26 | BASE_IMAGES_TAG_CONFIG = 'base-images-tag.json' 27 | 28 | # 29 | # See the Makefile for an in-depth explanation of how we're using the base 30 | # Docker images. 31 | # 32 | try: 33 | with open(os.path.join(main_dir, BASE_IMAGES_TAG_CONFIG)) as tag_config: 34 | tag_json = json.load(tag_config) 35 | BASE_IMAGES_TAG = tag_json['base_images_tag'] 36 | except KeyError: 37 | print "base_images_tag must be set in %s" % (BASE_IMAGES_TAG_CONFIG,) 38 | sys.exit(1) 39 | 40 | LOCAL_RESOURCES_DIR = os.path.join(prestoadmin.main_dir, 41 | 'tests/product/resources/') 42 | 43 | DEFAULT_DOCKER_MOUNT_POINT = '/mnt/presto-admin' 44 | DEFAULT_LOCAL_MOUNT_POINT = os.path.join(main_dir, 'tmp/docker-pa/') 45 | -------------------------------------------------------------------------------- /tests/product/mode_installers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ 17 | Installers for installing mode.json onto clusters 18 | """ 19 | 20 | import json 21 | 22 | from overrides import overrides 23 | from prestoadmin import config 24 | from prestoadmin.mode import VALID_MODES, MODE_KEY, MODE_STANDALONE 25 | from tests.base_installer import BaseInstaller 26 | from tests.product.config_dir_utils import get_mode_config_path 27 | 28 | 29 | class BaseModeInstaller(BaseInstaller): 30 | def __init__(self, testcase, mode): 31 | self.testcase = testcase 32 | testcase.assertIn(mode, VALID_MODES) 33 | self.mode = mode 34 | self.json = config.json_to_string(self._get_mode_cfg(self.mode)) 35 | 36 | @staticmethod 37 | def _get_mode_cfg(mode): 38 | return {MODE_KEY: mode} 39 | 40 | @staticmethod 41 | @overrides 42 | def get_dependencies(): 43 | return [] 44 | 45 | @overrides 46 | def install(self): 47 | self.testcase.cluster.write_content_to_host( 48 | self.json, get_mode_config_path(), self.testcase.cluster.master) 49 | 50 | @overrides 51 | def get_keywords(self, *args, **kwargs): 52 | return {} 53 | 54 | @staticmethod 55 | def _assert_installed(testcase, expected_mode): 56 | json_str = testcase.cluster.exec_cmd_on_host( 57 | testcase.cluster.master, 'cat %s' % get_mode_config_path()) 58 | 59 | actual_mode_cfg = json.loads(json_str) 60 | testcase.assertEqual( 61 | BaseModeInstaller._get_mode_cfg( 62 | expected_mode), actual_mode_cfg) 63 | 64 | 65 | class StandaloneModeInstaller(BaseModeInstaller): 66 | def __init__(self, testcase): 67 | super(StandaloneModeInstaller, self).__init__( 68 | testcase, MODE_STANDALONE) 69 | 70 | @staticmethod 71 | def assert_installed(testcase): 72 | BaseModeInstaller._assert_installed(testcase, MODE_STANDALONE) 73 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_config.txt: -------------------------------------------------------------------------------- 1 | 2 | master: Configuration file at /etc/presto/config.properties: 3 | coordinator=true 4 | discovery-server.enabled=true 5 | discovery.uri=http://master:7070 6 | http-server.http.port=7070 7 | node-scheduler.include-coordinator=false 8 | query.max-memory-per-node=512MB 9 | query.max-memory=50GB 10 | 11 | 12 | slave1: Configuration file at /etc/presto/config.properties: 13 | coordinator=false 14 | discovery.uri=http://master:7070 15 | http-server.http.port=7070 16 | query.max-memory-per-node=512MB 17 | query.max-memory=50GB 18 | 19 | 20 | slave2: Configuration file at /etc/presto/config.properties: 21 | coordinator=false 22 | discovery.uri=http://master:7070 23 | http-server.http.port=7070 24 | query.max-memory-per-node=512MB 25 | query.max-memory=50GB 26 | 27 | 28 | slave3: Configuration file at /etc/presto/config.properties: 29 | coordinator=false 30 | discovery.uri=http://master:7070 31 | http-server.http.port=7070 32 | query.max-memory-per-node=512MB 33 | query.max-memory=50GB 34 | 35 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_default.txt: -------------------------------------------------------------------------------- 1 | master: Configuration file at /etc/presto/node.properties: 2 | node.id=.* 3 | catalog.config-dir=/etc/presto/catalog 4 | node.data-dir=/var/lib/presto/data 5 | node.environment=presto 6 | node.launcher-log-file=/var/log/presto/launcher.log 7 | node.server-log-file=/var/log/presto/server.log 8 | plugin.dir=/usr/lib/presto/lib/plugin 9 | 10 | 11 | master: Configuration file at /etc/presto/jvm.config: 12 | -server 13 | -Xmx16G 14 | -XX:\-UseBiasedLocking 15 | -XX:\+UseG1GC 16 | -XX:G1HeapRegionSize=32M 17 | -XX:\+ExplicitGCInvokesConcurrent 18 | -XX:\+HeapDumpOnOutOfMemoryError 19 | -XX:\+UseGCOverheadLimit 20 | -XX:\+ExitOnOutOfMemoryError 21 | -XX:ReservedCodeCacheSize=512M 22 | -DHADOOP_USER_NAME=hive 23 | 24 | 25 | master: Configuration file at /etc/presto/config.properties: 26 | coordinator=true 27 | discovery-server.enabled=true 28 | discovery.uri=http://master:7070 29 | http-server.http.port=7070 30 | node.scheduler.include-coordinator=false 31 | query.max-memory-per-node=512MB 32 | query.max-memory=50GB 33 | 34 | 35 | slave1: Configuration file at /etc/presto/node.properties: 36 | node.id=.* 37 | catalog.config-dir=/etc/presto/catalog 38 | node.data-dir=/var/lib/presto/data 39 | node.environment=presto 40 | node.launcher-log-file=/var/log/presto/launcher.log 41 | node.server-log-file=/var/log/presto/server.log 42 | plugin.dir=/usr/lib/presto/lib/plugin 43 | 44 | 45 | slave1: Configuration file at /etc/presto/jvm.config: 46 | -server 47 | -Xmx16G 48 | -XX:\-UseBiasedLocking 49 | -XX:\+UseG1GC 50 | -XX:G1HeapRegionSize=32M 51 | -XX:\+ExplicitGCInvokesConcurrent 52 | -XX:\+HeapDumpOnOutOfMemoryError 53 | -XX:\+UseGCOverheadLimit 54 | -XX:\+ExitOnOutOfMemoryError 55 | -XX:ReservedCodeCacheSize=512M 56 | -DHADOOP_USER_NAME=hive 57 | 58 | 59 | slave1: Configuration file at /etc/presto/config.properties: 60 | coordinator=false 61 | discovery.uri=http://master:7070 62 | http-server.http.port=7070 63 | query.max-memory-per-node=512MB 64 | query.max-memory=50GB 65 | 66 | 67 | slave2: Configuration file at /etc/presto/node.properties: 68 | node.id=.* 69 | catalog.config-dir=/etc/presto/catalog 70 | node.data-dir=/var/lib/presto/data 71 | node.environment=presto 72 | node.launcher-log-file=/var/log/presto/launcher.log 73 | node.server-log-file=/var/log/presto/server.log 74 | plugin.dir=/usr/lib/presto/lib/plugin 75 | 76 | 77 | slave2: Configuration file at /etc/presto/jvm.config: 78 | -server 79 | -Xmx16G 80 | -XX:\-UseBiasedLocking 81 | -XX:\+UseG1GC 82 | -XX:G1HeapRegionSize=32M 83 | -XX:\+ExplicitGCInvokesConcurrent 84 | -XX:\+HeapDumpOnOutOfMemoryError 85 | -XX:\+UseGCOverheadLimit 86 | -XX:\+ExitOnOutOfMemoryError 87 | -XX:ReservedCodeCacheSize=512M 88 | -DHADOOP_USER_NAME=hive 89 | 90 | 91 | slave2: Configuration file at /etc/presto/config.properties: 92 | coordinator=false 93 | discovery.uri=http://master:7070 94 | http-server.http.port=7070 95 | query.max-memory-per-node=512MB 96 | query.max-memory=50GB 97 | 98 | 99 | slave3: Configuration file at /etc/presto/node.properties: 100 | node.id=.* 101 | catalog.config-dir=/etc/presto/catalog 102 | node.data-dir=/var/lib/presto/data 103 | node.environment=presto 104 | node.launcher-log-file=/var/log/presto/launcher.log 105 | node.server-log-file=/var/log/presto/server.log 106 | plugin.dir=/usr/lib/presto/lib/plugin 107 | 108 | 109 | slave3: Configuration file at /etc/presto/jvm.config: 110 | -server 111 | -Xmx16G 112 | -XX:\-UseBiasedLocking 113 | -XX:\+UseG1GC 114 | -XX:G1HeapRegionSize=32M 115 | -XX:\+ExplicitGCInvokesConcurrent 116 | -XX:\+HeapDumpOnOutOfMemoryError 117 | -XX:\+UseGCOverheadLimit 118 | -XX:\+ExitOnOutOfMemoryError 119 | -XX:ReservedCodeCacheSize=512M 120 | -DHADOOP_USER_NAME=hive 121 | 122 | 123 | slave3: Configuration file at /etc/presto/config.properties: 124 | coordinator=false 125 | discovery.uri=http://master:7070 126 | http-server.http.port=7070 127 | query.max-memory-per-node=512MB 128 | query.max-memory=50GB 129 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_default_master_slave1.txt: -------------------------------------------------------------------------------- 1 | master: Configuration file at /etc/presto/node.properties: 2 | node.id=.* 3 | catalog.config-dir=/etc/presto/catalog 4 | node.data-dir=/var/lib/presto/data 5 | node.environment=presto 6 | node.launcher-log-file=/var/log/presto/launcher.log 7 | node.server-log-file=/var/log/presto/server.log 8 | plugin.dir=/usr/lib/presto/lib/plugin 9 | 10 | 11 | master: Configuration file at /etc/presto/jvm.config: 12 | -server 13 | -Xmx16G 14 | -XX:\-UseBiasedLocking 15 | -XX:\+UseG1GC 16 | -XX:G1HeapRegionSize=32M 17 | -XX:\+ExplicitGCInvokesConcurrent 18 | -XX:\+HeapDumpOnOutOfMemoryError 19 | -XX:\+UseGCOverheadLimit 20 | -XX:\+ExitOnOutOfMemoryError 21 | -XX:ReservedCodeCacheSize=512M 22 | -DHADOOP_USER_NAME=hive 23 | 24 | 25 | master: Configuration file at /etc/presto/config.properties: 26 | coordinator=true 27 | discovery-server.enabled=true 28 | discovery.uri=http://master:7070 29 | http-server.http.port=7070 30 | node.scheduler.include-coordinator=false 31 | query.max-memory-per-node=512MB 32 | query.max-memory=50GB 33 | 34 | 35 | slave1: Configuration file at /etc/presto/node.properties: 36 | node.id=.* 37 | catalog.config-dir=/etc/presto/catalog 38 | node.data-dir=/var/lib/presto/data 39 | node.environment=presto 40 | node.launcher-log-file=/var/log/presto/launcher.log 41 | node.server-log-file=/var/log/presto/server.log 42 | plugin.dir=/usr/lib/presto/lib/plugin 43 | 44 | 45 | slave1: Configuration file at /etc/presto/jvm.config: 46 | -server 47 | -Xmx16G 48 | -XX:\-UseBiasedLocking 49 | -XX:\+UseG1GC 50 | -XX:G1HeapRegionSize=32M 51 | -XX:\+ExplicitGCInvokesConcurrent 52 | -XX:\+HeapDumpOnOutOfMemoryError 53 | -XX:\+UseGCOverheadLimit 54 | -XX:\+ExitOnOutOfMemoryError 55 | -XX:ReservedCodeCacheSize=512M 56 | -DHADOOP_USER_NAME=hive 57 | 58 | 59 | slave1: Configuration file at /etc/presto/config.properties: 60 | coordinator=false 61 | discovery.uri=http://master:7070 62 | http-server.http.port=7070 63 | query.max-memory-per-node=512MB 64 | query.max-memory=50GB 65 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_default_slave2_slave3.txt: -------------------------------------------------------------------------------- 1 | slave2: Configuration file at /etc/presto/node.properties: 2 | node.id=.* 3 | catalog.config-dir=/etc/presto/catalog 4 | node.data-dir=/var/lib/presto/data 5 | node.environment=presto 6 | node.launcher-log-file=/var/log/presto/launcher.log 7 | node.server-log-file=/var/log/presto/server.log 8 | plugin.dir=/usr/lib/presto/lib/plugin 9 | 10 | 11 | slave2: Configuration file at /etc/presto/jvm.config: 12 | -server 13 | -Xmx16G 14 | -XX:\-UseBiasedLocking 15 | -XX:\+UseG1GC 16 | -XX:G1HeapRegionSize=32M 17 | -XX:\+ExplicitGCInvokesConcurrent 18 | -XX:\+HeapDumpOnOutOfMemoryError 19 | -XX:\+UseGCOverheadLimit 20 | -XX:\+ExitOnOutOfMemoryError 21 | -XX:ReservedCodeCacheSize=512M 22 | -DHADOOP_USER_NAME=hive 23 | 24 | 25 | slave2: Configuration file at /etc/presto/config.properties: 26 | coordinator=false 27 | discovery.uri=http://master:7070 28 | http-server.http.port=7070 29 | query.max-memory-per-node=512MB 30 | query.max-memory=50GB 31 | 32 | 33 | slave3: Configuration file at /etc/presto/node.properties: 34 | node.id=.* 35 | catalog.config-dir=/etc/presto/catalog 36 | node.data-dir=/var/lib/presto/data 37 | node.environment=presto 38 | node.launcher-log-file=/var/log/presto/launcher.log 39 | node.server-log-file=/var/log/presto/server.log 40 | plugin.dir=/usr/lib/presto/lib/plugin 41 | 42 | 43 | slave3: Configuration file at /etc/presto/jvm.config: 44 | -server 45 | -Xmx16G 46 | -XX:\-UseBiasedLocking 47 | -XX:\+UseG1GC 48 | -XX:G1HeapRegionSize=32M 49 | -XX:\+ExplicitGCInvokesConcurrent 50 | -XX:\+HeapDumpOnOutOfMemoryError 51 | -XX:\+UseGCOverheadLimit 52 | -XX:\+ExitOnOutOfMemoryError 53 | -XX:ReservedCodeCacheSize=512M 54 | -DHADOOP_USER_NAME=hive 55 | 56 | 57 | slave3: Configuration file at /etc/presto/config.properties: 58 | coordinator=false 59 | discovery.uri=http://master:7070 60 | http-server.http.port=7070 61 | query.max-memory-per-node=512MB 62 | query.max-memory=50GB 63 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_down_node.txt: -------------------------------------------------------------------------------- 1 | 2 | master: Configuration file at /etc/presto/config.properties: 3 | coordinator=false 4 | discovery.uri=http://.*:7070 5 | http-server.http.port=7070 6 | query.max-memory-per-node=512MB 7 | query.max-memory=50GB 8 | 9 | 10 | slave2: Configuration file at /etc/presto/config.properties: 11 | coordinator=false 12 | discovery.uri=http://.*:7070 13 | http-server.http.port=7070 14 | query.max-memory-per-node=512MB 15 | query.max-memory=50GB 16 | 17 | 18 | slave3: Configuration file at /etc/presto/config.properties: 19 | coordinator=false 20 | discovery.uri=http://.*:7070 21 | http-server.http.port=7070 22 | query.max-memory-per-node=512MB 23 | query.max-memory=50GB 24 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_jvm.txt: -------------------------------------------------------------------------------- 1 | 2 | master: Configuration file at /etc/presto/jvm.config: 3 | -server 4 | -Xmx16G 5 | -XX:-UseBiasedLocking 6 | -XX:+UseG1GC 7 | -XX:G1HeapRegionSize=32M 8 | -XX:+ExplicitGCInvokesConcurrent 9 | -XX:+HeapDumpOnOutOfMemoryError 10 | -XX:+UseGCOverheadLimit 11 | -XX:+ExitOnOutOfMemoryError 12 | -XX:ReservedCodeCacheSize=512M 13 | -DHADOOP_USER_NAME=hive 14 | 15 | 16 | slave1: Configuration file at /etc/presto/jvm.config: 17 | -server 18 | -Xmx16G 19 | -XX:-UseBiasedLocking 20 | -XX:+UseG1GC 21 | -XX:G1HeapRegionSize=32M 22 | -XX:+ExplicitGCInvokesConcurrent 23 | -XX:+HeapDumpOnOutOfMemoryError 24 | -XX:+UseGCOverheadLimit 25 | -XX:+ExitOnOutOfMemoryError 26 | -XX:ReservedCodeCacheSize=512M 27 | -DHADOOP_USER_NAME=hive 28 | 29 | 30 | slave2: Configuration file at /etc/presto/jvm.config: 31 | -server 32 | -Xmx16G 33 | -XX:-UseBiasedLocking 34 | -XX:+UseG1GC 35 | -XX:G1HeapRegionSize=32M 36 | -XX:+ExplicitGCInvokesConcurrent 37 | -XX:+HeapDumpOnOutOfMemoryError 38 | -XX:+UseGCOverheadLimit 39 | -XX:+ExitOnOutOfMemoryError 40 | -XX:ReservedCodeCacheSize=512M 41 | -DHADOOP_USER_NAME=hive 42 | 43 | 44 | slave3: Configuration file at /etc/presto/jvm.config: 45 | -server 46 | -Xmx16G 47 | -XX:-UseBiasedLocking 48 | -XX:+UseG1GC 49 | -XX:G1HeapRegionSize=32M 50 | -XX:+ExplicitGCInvokesConcurrent 51 | -XX:+HeapDumpOnOutOfMemoryError 52 | -XX:+UseGCOverheadLimit 53 | -XX:+ExitOnOutOfMemoryError 54 | -XX:ReservedCodeCacheSize=512M 55 | -DHADOOP_USER_NAME=hive 56 | 57 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_log.txt: -------------------------------------------------------------------------------- 1 | 2 | master: Configuration file at /etc/presto/log.properties: 3 | com.facebook.presto=WARN 4 | 5 | 6 | slave1: Configuration file at /etc/presto/log.properties: 7 | com.facebook.presto=WARN 8 | 9 | 10 | slave2: Configuration file at /etc/presto/log.properties: 11 | com.facebook.presto=WARN 12 | 13 | 14 | slave3: Configuration file at /etc/presto/log.properties: 15 | com.facebook.presto=WARN 16 | 17 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_log_none.txt: -------------------------------------------------------------------------------- 1 | 2 | Warning: [master] No configuration file found for master at /etc/presto/log.properties 3 | 4 | 5 | Warning: [slave1] No configuration file found for slave1 at /etc/presto/log.properties 6 | 7 | 8 | Warning: [slave2] No configuration file found for slave2 at /etc/presto/log.properties 9 | 10 | 11 | Warning: [slave3] No configuration file found for slave3 at /etc/presto/log.properties 12 | 13 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_node.txt: -------------------------------------------------------------------------------- 1 | master: Configuration file at /etc/presto/node.properties: 2 | node.id=.* 3 | catalog.config-dir=/etc/presto/catalog 4 | node.data-dir=/var/lib/presto/data 5 | node.environment=presto 6 | node.launcher-log-file=/var/log/presto/launcher.log 7 | node.server-log-file=/var/log/presto/server.log 8 | plugin.dir=/usr/lib/presto/lib/plugin 9 | 10 | 11 | slave1: Configuration file at /etc/presto/node.properties: 12 | node.id=.* 13 | catalog.config-dir=/etc/presto/catalog 14 | node.data-dir=/var/lib/presto/data 15 | node.environment=presto 16 | node.launcher-log-file=/var/log/presto/launcher.log 17 | node.server-log-file=/var/log/presto/server.log 18 | plugin.dir=/usr/lib/presto/lib/plugin 19 | 20 | 21 | slave2: Configuration file at /etc/presto/node.properties: 22 | node.id=.* 23 | catalog.config-dir=/etc/presto/catalog 24 | node.data-dir=/var/lib/presto/data 25 | node.environment=presto 26 | node.launcher-log-file=/var/log/presto/launcher.log 27 | node.server-log-file=/var/log/presto/server.log 28 | plugin.dir=/usr/lib/presto/lib/plugin 29 | 30 | 31 | slave3: Configuration file at /etc/presto/node.properties: 32 | node.id=.* 33 | catalog.config-dir=/etc/presto/catalog 34 | node.data-dir=/var/lib/presto/data 35 | node.environment=presto 36 | node.launcher-log-file=/var/log/presto/launcher.log 37 | node.server-log-file=/var/log/presto/server.log 38 | plugin.dir=/usr/lib/presto/lib/plugin 39 | -------------------------------------------------------------------------------- /tests/product/resources/configuration_show_none.txt: -------------------------------------------------------------------------------- 1 | 2 | Warning: [master] No configuration file found for master at /etc/presto/node.properties 3 | 4 | 5 | Warning: [master] No configuration file found for master at /etc/presto/jvm.config 6 | 7 | 8 | Warning: [master] No configuration file found for master at /etc/presto/config.properties 9 | 10 | 11 | Warning: [slave1] No configuration file found for slave1 at /etc/presto/node.properties 12 | 13 | 14 | Warning: [slave1] No configuration file found for slave1 at /etc/presto/jvm.config 15 | 16 | 17 | Warning: [slave1] No configuration file found for slave1 at /etc/presto/config.properties 18 | 19 | 20 | Warning: [slave2] No configuration file found for slave2 at /etc/presto/node.properties 21 | 22 | 23 | Warning: [slave2] No configuration file found for slave2 at /etc/presto/jvm.config 24 | 25 | 26 | Warning: [slave2] No configuration file found for slave2 at /etc/presto/config.properties 27 | 28 | 29 | Warning: [slave3] No configuration file found for slave3 at /etc/presto/node.properties 30 | 31 | 32 | Warning: [slave3] No configuration file found for slave3 at /etc/presto/jvm.config 33 | 34 | 35 | Warning: [slave3] No configuration file found for slave3 at /etc/presto/config.properties 36 | 37 | -------------------------------------------------------------------------------- /tests/product/resources/dummy-rpm.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/product/resources/dummy-rpm.rpm -------------------------------------------------------------------------------- /tests/product/resources/install-admin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | set -e 16 | 17 | cp /{mount_dir}/prestoadmin-*.tar.gz ~ 18 | cd ~ 19 | tar -zxf prestoadmin-*.tar.gz 20 | cd prestoadmin 21 | ./install-prestoadmin.sh 22 | -------------------------------------------------------------------------------- /tests/product/resources/install_twice.txt: -------------------------------------------------------------------------------- 1 | Using rpm_specifier as a local path 2 | Fetching local presto rpm at path: .* 3 | Found existing rpm at: .* 4 | 5 | Fatal error: [%(slave2)s] sudo() received nonzero return code 1 while executing! 6 | 7 | Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s 8 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" 9 | 10 | Aborting. 11 | Deploying rpm on %(slave2)s... 12 | Package deployed successfully on: %(slave2)s 13 | [%(slave2)s] out: package %(rpm_basename)s is already installed 14 | [%(slave2)s] out: 15 | 16 | Fatal error: [%(master)s] sudo() received nonzero return code 1 while executing! 17 | 18 | Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s 19 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" 20 | 21 | Aborting. 22 | Deploying rpm on %(master)s... 23 | Package deployed successfully on: %(master)s 24 | [%(master)s] out: package %(rpm_basename)s is already installed 25 | [%(master)s] out: 26 | 27 | Fatal error: [%(slave3)s] sudo() received nonzero return code 1 while executing! 28 | 29 | Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s 30 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" 31 | 32 | Aborting. 33 | Deploying rpm on %(slave3)s... 34 | Package deployed successfully on: %(slave3)s 35 | [%(slave3)s] out: package %(rpm_basename)s is already installed 36 | [%(slave3)s] out: 37 | 38 | Fatal error: [%(slave1)s] sudo() received nonzero return code 1 while executing! 39 | 40 | Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s 41 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" 42 | 43 | Aborting. 44 | Deploying rpm on %(slave1)s... 45 | Package deployed successfully on: %(slave1)s 46 | [%(slave1)s] out: package %(rpm_basename)s is already installed 47 | [%(slave1)s] out: -------------------------------------------------------------------------------- /tests/product/resources/invalid_json.json: -------------------------------------------------------------------------------- 1 | { 2 | "user": "root" 3 | bad json!!! 4 | } 5 | -------------------------------------------------------------------------------- /tests/product/resources/non_root_sudo_warning_text.txt: -------------------------------------------------------------------------------- 1 | [master] out: 2 | [master] out: 3 | [master] out: 4 | [master] out: 5 | [master] out: #1) Respect the privacy of others. 6 | [master] out: #2) Think before you type. 7 | [master] out: #3) With great power comes great responsibility. 8 | [master] out: Administrator. It usually boils down to these three things: 9 | [master] out: We trust you have received the usual lecture from the local System 10 | [master] out: sudo password: 11 | [slave1] out: 12 | [slave1] out: 13 | [slave1] out: 14 | [slave1] out: 15 | [slave1] out: #1) Respect the privacy of others. 16 | [slave1] out: #2) Think before you type. 17 | [slave1] out: #3) With great power comes great responsibility. 18 | [slave1] out: Administrator. It usually boils down to these three things: 19 | [slave1] out: We trust you have received the usual lecture from the local System 20 | [slave1] out: sudo password: 21 | [slave2] out: 22 | [slave2] out: 23 | [slave2] out: 24 | [slave2] out: 25 | [slave2] out: #1) Respect the privacy of others. 26 | [slave2] out: #2) Think before you type. 27 | [slave2] out: #3) With great power comes great responsibility. 28 | [slave2] out: Administrator. It usually boils down to these three things: 29 | [slave2] out: We trust you have received the usual lecture from the local System 30 | [slave2] out: sudo password: 31 | [slave3] out: 32 | [slave3] out: 33 | [slave3] out: 34 | [slave3] out: 35 | [slave3] out: #1) Respect the privacy of others. 36 | [slave3] out: #2) Think before you type. 37 | [slave3] out: #3) With great power comes great responsibility. 38 | [slave3] out: Administrator. It usually boils down to these three things: 39 | [slave3] out: We trust you have received the usual lecture from the local System 40 | [slave3] out: sudo password: 41 | -------------------------------------------------------------------------------- /tests/product/resources/non_sudo_uninstall.txt: -------------------------------------------------------------------------------- 1 | 2 | Fatal error: [slave3] sudo() received nonzero return code 1 while executing! 3 | 4 | Requested: set -m; /etc/init.d/presto stop 5 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" 6 | 7 | Aborting. 8 | [slave3] out: 9 | [slave3] out: We trust you have received the usual lecture from the local System 10 | [slave3] out: Administrator. It usually boils down to these three things: 11 | [slave3] out: 12 | [slave3] out: #1) Respect the privacy of others. 13 | [slave3] out: #2) Think before you type. 14 | [slave3] out: #3) With great power comes great responsibility. 15 | [slave3] out: 16 | [slave3] out: sudo password: 17 | [slave3] out: testuser is not in the sudoers file. This incident will be reported. 18 | [slave3] out: 19 | 20 | Fatal error: [slave1] sudo() received nonzero return code 1 while executing! 21 | 22 | Requested: set -m; /etc/init.d/presto stop 23 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" 24 | 25 | Aborting. 26 | [slave1] out: 27 | [slave1] out: We trust you have received the usual lecture from the local System 28 | [slave1] out: Administrator. It usually boils down to these three things: 29 | [slave1] out: 30 | [slave1] out: #1) Respect the privacy of others. 31 | [slave1] out: #2) Think before you type. 32 | [slave1] out: #3) With great power comes great responsibility. 33 | [slave1] out: 34 | [slave1] out: sudo password: 35 | [slave1] out: testuser is not in the sudoers file. This incident will be reported. 36 | [slave1] out: 37 | 38 | Fatal error: [slave2] sudo() received nonzero return code 1 while executing! 39 | 40 | Requested: set -m; /etc/init.d/presto stop 41 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" 42 | 43 | Aborting. 44 | [slave2] out: 45 | [slave2] out: We trust you have received the usual lecture from the local System 46 | [slave2] out: Administrator. It usually boils down to these three things: 47 | [slave2] out: 48 | [slave2] out: #1) Respect the privacy of others. 49 | [slave2] out: #2) Think before you type. 50 | [slave2] out: #3) With great power comes great responsibility. 51 | [slave2] out: 52 | [slave2] out: sudo password: 53 | [slave2] out: testuser is not in the sudoers file. This incident will be reported. 54 | [slave2] out: 55 | 56 | Fatal error: [master] sudo() received nonzero return code 1 while executing! 57 | 58 | Requested: set -m; /etc/init.d/presto stop 59 | Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" 60 | 61 | Aborting. 62 | [master] out: 63 | [master] out: We trust you have received the usual lecture from the local System 64 | [master] out: Administrator. It usually boils down to these three things: 65 | [master] out: 66 | [master] out: #1) Respect the privacy of others. 67 | [master] out: #2) Think before you type. 68 | [master] out: #3) With great power comes great responsibility. 69 | [master] out: 70 | [master] out: sudo password: 71 | [master] out: testuser is not in the sudoers file. This incident will be reported. 72 | [master] out: 73 | -------------------------------------------------------------------------------- /tests/product/resources/parallel_password_failure.txt: -------------------------------------------------------------------------------- 1 | 2 | Fatal error: [%(slave2)s] Needed to prompt for a connection or sudo password (host: %(slave2)s), but input would be ambiguous in parallel mode 3 | 4 | Aborting. 5 | Deploying tpch.properties catalog configurations on: %(slave2)s 6 | 7 | Fatal error: [%(slave1)s] Needed to prompt for a connection or sudo password (host: %(slave1)s), but input would be ambiguous in parallel mode 8 | 9 | Aborting. 10 | Deploying tpch.properties catalog configurations on: %(slave1)s 11 | 12 | Fatal error: [%(master)s] Needed to prompt for a connection or sudo password (host: %(master)s), but input would be ambiguous in parallel mode 13 | 14 | Aborting. 15 | Deploying tpch.properties catalog configurations on: %(master)s 16 | 17 | Fatal error: [%(slave3)s] Needed to prompt for a connection or sudo password (host: %(slave3)s), but input would be ambiguous in parallel mode 18 | 19 | Aborting. 20 | Deploying tpch.properties catalog configurations on: %(slave3)s 21 | -------------------------------------------------------------------------------- /tests/product/resources/slider-assembly-0.80.0-incubating-all.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/product/resources/slider-assembly-0.80.0-incubating-all.tar.gz -------------------------------------------------------------------------------- /tests/product/resources/uninstall_twice.txt: -------------------------------------------------------------------------------- 1 | 2 | Warning: [slave2] Presto is not installed. 3 | 4 | 5 | Warning: [master] Presto is not installed. 6 | 7 | 8 | Warning: [slave3] Presto is not installed. 9 | 10 | 11 | Warning: [slave1] Presto is not installed. 12 | 13 | 14 | Fatal error: [slave2] Unable to uninstall package on: slave2 15 | 16 | Aborting. 17 | 18 | Fatal error: [slave3] Unable to uninstall package on: slave3 19 | 20 | Aborting. 21 | 22 | Fatal error: [master] Unable to uninstall package on: master 23 | 24 | Aborting. 25 | 26 | Fatal error: [slave1] Unable to uninstall package on: slave1 27 | 28 | Aborting. 29 | -------------------------------------------------------------------------------- /tests/product/standalone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/product/standalone/__init__.py -------------------------------------------------------------------------------- /tests/product/test_error_handling.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | System tests for error handling in presto-admin 17 | """ 18 | from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider 19 | from tests.product.base_product_case import BaseProductTestCase 20 | from tests.product.cluster_types import STANDALONE_PA_CLUSTER 21 | 22 | 23 | class TestErrorHandling(BaseProductTestCase): 24 | 25 | def setUp(self): 26 | super(TestErrorHandling, self).setUp() 27 | self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER) 28 | self.upload_topology() 29 | 30 | def test_wrong_arguments_parallel(self): 31 | actual = self.run_prestoadmin('server start extra_arg', 32 | raise_error=False) 33 | expected = "Incorrect number of arguments to task.\n\n" \ 34 | "Displaying detailed information for task " \ 35 | "'server start':\n\n Start the Presto server on all " \ 36 | "nodes\n \n A status check is performed on the " \ 37 | "entire cluster and a list of\n servers that did not " \ 38 | "start, if any, are reported at the end.\n\n" 39 | self.assertEqual(expected, actual) 40 | 41 | def test_wrong_arguments_serial(self): 42 | actual = self.run_prestoadmin('server start extra_arg --serial', 43 | raise_error=False) 44 | expected = "Incorrect number of arguments to task.\n\n" \ 45 | "Displaying detailed information for task " \ 46 | "'server start':\n\n Start the Presto server on all " \ 47 | "nodes\n \n A status check is performed on the " \ 48 | "entire cluster and a list of\n servers that did not " \ 49 | "start, if any, are reported at the end.\n\n" 50 | self.assertEqual(expected, actual) 51 | -------------------------------------------------------------------------------- /tests/product/test_server_uninstall.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from nose.plugins.attrib import attr 18 | 19 | from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider 20 | from tests.product.base_product_case import BaseProductTestCase 21 | from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER 22 | from tests.product.constants import LOCAL_RESOURCES_DIR 23 | from tests.product.standalone.presto_installer import StandalonePrestoInstaller 24 | 25 | uninstall_output = ['Package uninstalled successfully on: slave1', 26 | 'Package uninstalled successfully on: slave2', 27 | 'Package uninstalled successfully on: slave3', 28 | 'Package uninstalled successfully on: master'] 29 | 30 | 31 | class TestServerUninstall(BaseProductTestCase): 32 | def setUp(self): 33 | super(TestServerUninstall, self).setUp() 34 | self.installer = StandalonePrestoInstaller(self) 35 | 36 | @attr('smoketest') 37 | def test_uninstall(self): 38 | self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) 39 | start_output = self.run_prestoadmin('server start') 40 | process_per_host = self.get_process_per_host(start_output.splitlines()) 41 | self.assert_started(process_per_host) 42 | 43 | cmd_output = self.run_prestoadmin( 44 | 'server uninstall', raise_error=False).splitlines() 45 | self.assert_stopped(process_per_host) 46 | expected = uninstall_output + self.expected_stop()[:] 47 | self.assertRegexpMatchesLineByLine(cmd_output, expected) 48 | 49 | for container in self.cluster.all_hosts(): 50 | self.assert_uninstalled_dirs_removed(container) 51 | 52 | def assert_uninstalled_dirs_removed(self, container): 53 | self.installer.assert_uninstalled(container) 54 | self.assert_path_removed(container, '/etc/presto') 55 | self.assert_path_removed(container, '/usr/lib/presto') 56 | self.assert_path_removed(container, '/var/lib/presto') 57 | self.assert_path_removed(container, '/usr/shared/doc/presto') 58 | self.assert_path_removed(container, '/etc/init.d/presto') 59 | 60 | def test_uninstall_twice(self): 61 | self.test_uninstall() 62 | 63 | output = self.run_prestoadmin('server uninstall', raise_error=False) 64 | with open(os.path.join(LOCAL_RESOURCES_DIR, 'uninstall_twice.txt'), 65 | 'r') as f: 66 | expected = f.read() 67 | 68 | self.assertEqualIgnoringOrder(expected, output) 69 | 70 | def test_uninstall_lost_host(self): 71 | self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) 72 | 73 | self.cluster.stop_host( 74 | self.cluster.slaves[0]) 75 | 76 | expected = self.down_node_connection_error( 77 | self.cluster.internal_slaves[0]) 78 | cmd_output = self.run_prestoadmin('server uninstall', 79 | raise_error=False) 80 | self.assertRegexpMatches(cmd_output, expected) 81 | 82 | for container in [self.cluster.internal_master, 83 | self.cluster.internal_slaves[1], 84 | self.cluster.internal_slaves[2]]: 85 | self.assert_uninstalled_dirs_removed(container) 86 | -------------------------------------------------------------------------------- /tests/product/test_topology.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from nose.plugins.attrib import attr 18 | 19 | from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider 20 | from tests.product.base_product_case import BaseProductTestCase 21 | from tests.product.cluster_types import STANDALONE_PA_CLUSTER 22 | from tests.product.config_dir_utils import get_config_file_path 23 | from tests.product.constants import LOCAL_RESOURCES_DIR 24 | 25 | 26 | topology_with_slave1_coord = """{{'coordinator': u'slave1', 27 | 'port': 22, 28 | 'username': '{user}', 29 | 'workers': [u'master', 30 | u'slave2', 31 | u'slave3']}} 32 | """ 33 | 34 | normal_topology = """{{'coordinator': u'master', 35 | 'port': 22, 36 | 'username': '{user}', 37 | 'workers': [u'slave1', 38 | u'slave2', 39 | u'slave3']}} 40 | """ 41 | 42 | local_topology = """{{'coordinator': 'localhost', 43 | 'port': 22, 44 | 'username': '{user}', 45 | 'workers': ['localhost']}} 46 | """ 47 | 48 | 49 | class TestTopologyShow(BaseProductTestCase): 50 | 51 | def setUp(self): 52 | super(TestTopologyShow, self).setUp() 53 | self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER) 54 | 55 | @attr('smoketest') 56 | def test_topology_show(self): 57 | self.upload_topology() 58 | actual = self.run_prestoadmin('topology show') 59 | expected = normal_topology.format(user=self.cluster.user) 60 | self.assertEqual(expected, actual) 61 | 62 | def test_topology_show_empty_config(self): 63 | self.dump_and_cp_topology(topology={}) 64 | actual = self.run_prestoadmin('topology show') 65 | self.assertEqual(local_topology.format(user=self.cluster.user), actual) 66 | 67 | def test_topology_show_bad_json(self): 68 | self.cluster.copy_to_host( 69 | os.path.join(LOCAL_RESOURCES_DIR, 'invalid_json.json'), 70 | self.cluster.master 71 | ) 72 | self.cluster.exec_cmd_on_host( 73 | self.cluster.master, 74 | 'cp %s %s' % 75 | (os.path.join(self.cluster.mount_dir, 'invalid_json.json'), get_config_file_path()) 76 | ) 77 | self.assertRaisesRegexp(OSError, 78 | 'Expecting , delimiter: line 3 column 3 ' 79 | '\(char 21\) More detailed information ' 80 | 'can be found in ' 81 | '.*/.prestoadmin/log/presto-admin.log\n', 82 | self.run_prestoadmin, 83 | 'topology show') 84 | -------------------------------------------------------------------------------- /tests/product/timing_test_decorator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | from time import time 5 | 6 | 7 | logger = logging.getLogger() 8 | logger.setLevel(logging.INFO) 9 | message_format = '%(levelname)s - %(message)s' 10 | formatter = logging.Formatter(message_format) 11 | console_handler = logging.StreamHandler(sys.stdout) 12 | console_handler.setLevel(logging.INFO) 13 | console_handler.setFormatter(formatter) 14 | logger.addHandler(console_handler) 15 | 16 | 17 | def log_function_time(): 18 | """ 19 | Returns: Prints the execution time of the decorated function to the 20 | console. If the execution time exceeds 10 minutes, it will use 'error' 21 | for the message level. Otherwise, it will use 'info'. 22 | """ 23 | def name_wrapper(function): 24 | def time_wrapper(*args, **kwargs): 25 | global logger 26 | function_name = function.__name__ 27 | 28 | start_time = time() 29 | return_value = function(*args, **kwargs) 30 | elapsed_time = time() - start_time 31 | 32 | travis_output_time_limit = 600 33 | message_level = logging.ERROR if elapsed_time >= travis_output_time_limit \ 34 | else logging.INFO 35 | logging.disable(logging.NOTSET) 36 | logger.log(message_level, 37 | "%s completed in %s seconds...", 38 | function_name, 39 | str(elapsed_time)) 40 | logging.disable(logging.CRITICAL) 41 | 42 | return return_value 43 | return time_wrapper 44 | return name_wrapper 45 | -------------------------------------------------------------------------------- /tests/product/topology_installer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module for setting the topology on the presto-admin host prior to installing 17 | presto 18 | """ 19 | 20 | from tests.base_installer import BaseInstaller 21 | from tests.product.config_dir_utils import get_config_file_path 22 | 23 | 24 | class TopologyInstaller(BaseInstaller): 25 | def __init__(self, testcase): 26 | self.testcase = testcase 27 | 28 | @staticmethod 29 | def get_dependencies(): 30 | return [] 31 | 32 | def install(self): 33 | self.testcase.upload_topology(cluster=self.testcase.cluster) 34 | 35 | @staticmethod 36 | def assert_installed(testcase, msg=None): 37 | testcase.cluster.exec_cmd_on_host( 38 | testcase.cluster.master, 39 | 'test -r %s' % get_config_file_path()) 40 | 41 | def get_keywords(self): 42 | return {} 43 | -------------------------------------------------------------------------------- /tests/rpm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/rpm/test_rpm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Product tests for generating an online and offline installer for presto-admin 17 | """ 18 | 19 | import os 20 | from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider 21 | from tests.product.base_product_case import BaseProductTestCase, docker_only 22 | from tests.product.cluster_types import STANDALONE_PA_CLUSTER, STANDALONE_PRESTO_CLUSTER 23 | from tests.product.standalone.presto_installer import StandalonePrestoInstaller 24 | from tests.product.test_server_install import relocate_jdk_directory 25 | 26 | 27 | class TestRpm(BaseProductTestCase): 28 | def setUp(self): 29 | super(TestRpm, self).setUp() 30 | self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER) 31 | 32 | @docker_only 33 | def test_install_fails_java8_not_found(self): 34 | installer = StandalonePrestoInstaller(self) 35 | with relocate_jdk_directory(self.cluster, '/usr'): 36 | self.upload_topology() 37 | cmd_output = installer.install(pa_raise_error=False) 38 | actual = cmd_output.splitlines() 39 | num_failures = 0 40 | for line in enumerate(actual): 41 | if str(line).find('Error: Required Java version' 42 | ' could not be found') != -1: 43 | num_failures += 1 44 | 45 | self.assertEqual(4, num_failures) 46 | 47 | for container in self.cluster.all_hosts(): 48 | installer.assert_uninstalled(container) 49 | 50 | @docker_only 51 | def test_server_starts_java8_in_bin_java(self): 52 | installer = StandalonePrestoInstaller(self) 53 | 54 | with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: 55 | java_bin = os.path.join(new_java_home, 'bin', 'java') 56 | 57 | for container in self.cluster.all_hosts(): 58 | self.cluster.exec_cmd_on_host( 59 | container, 'ln -s %s /bin/java' % (java_bin,)) 60 | 61 | self.upload_topology() 62 | 63 | installer.install() 64 | 65 | # starts successfully with java8_home set 66 | output = self.run_prestoadmin('server start') 67 | self.assertFalse( 68 | 'Warning: No value found for JAVA8_HOME. Default Java will be ' 69 | 'used.' in output) 70 | 71 | @docker_only 72 | def test_server_starts_no_java8_variable(self): 73 | self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) 74 | self.run_script_from_prestoadmin_dir('rm /etc/presto/env.sh') 75 | # tests that no error is encountered 76 | self.run_prestoadmin('server start') 77 | 78 | @docker_only 79 | def test_started_with_presto_user(self): 80 | self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) 81 | start_output = self.run_prestoadmin('server start').splitlines() 82 | process_per_host = self.get_process_per_host(start_output) 83 | 84 | for host, pid in process_per_host: 85 | user_for_pid = self.run_script_from_prestoadmin_dir( 86 | 'uid=$(awk \'/^Uid:/{print $2}\' /proc/%s/status);' 87 | 'getent passwd "$uid" | awk -F: \'{print $1}\'' % pid, 88 | host) 89 | self.assertEqual(user_for_pid.strip(), 'presto') 90 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | class SudoResult(object): 2 | def __init__(self): 3 | super(SudoResult, self).__init__() 4 | self.return_code = 0 5 | self.failed = False 6 | -------------------------------------------------------------------------------- /tests/unit/base_unit_case.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mock import patch 16 | 17 | from prestoadmin.standalone.config import StandaloneConfig 18 | from prestoadmin.util.presto_config import PrestoConfig 19 | 20 | from tests.base_test_case import BaseTestCase 21 | 22 | PRESTO_CONFIG = PrestoConfig({ 23 | 'http-server.http.enabled': 'true', 24 | 'http-server.https.enabled': 'false', 25 | 'http-server.http.port': '8080', 26 | 'http-server.https.port': '7878', 27 | 'http-server.https.keystore.path': '/UPDATE/THIS/PATH', 28 | 'http-server.https.keystore.key': 'UPDATE PASSWORD'}, 29 | "TEST_PATH", 30 | "TEST_HOST") 31 | 32 | 33 | class BaseUnitCase(BaseTestCase): 34 | 35 | ''' 36 | Tasks generally require that the configuration they need to run has been 37 | loaded. This takes care of loading the config without going to the 38 | filesystem. For cases where you want to test the configuration load process 39 | itself, you should pass load_config=False to setUp. 40 | ''' 41 | def setUp(self, capture_output=False, load_config=True): 42 | super(BaseUnitCase, self).setUp(capture_output=capture_output) 43 | if load_config: 44 | @patch('tests.unit.base_unit_case.StandaloneConfig.' 45 | '_get_conf_from_file') 46 | def loader(mock_get_conf): 47 | mock_get_conf.return_value = {'username': 'user', 48 | 'port': 1234, 49 | 'coordinator': 'master', 50 | 'workers': ['slave1', 'slave2']} 51 | 52 | config = StandaloneConfig() 53 | config.get_config() 54 | loader() 55 | -------------------------------------------------------------------------------- /tests/unit/resources/empty.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/unit/resources/empty.txt -------------------------------------------------------------------------------- /tests/unit/resources/invalid.properties: -------------------------------------------------------------------------------- 1 | abcd -------------------------------------------------------------------------------- /tests/unit/resources/invalid_json_conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "user": "me" 3 | Invalid!!! 4 | } 5 | -------------------------------------------------------------------------------- /tests/unit/resources/server_status_out.txt: -------------------------------------------------------------------------------- 1 | Server Status: 2 | Node1(IP: IP1, Roles: coordinator, worker): Running 3 | Node URI(http): http://active/statement 4 | Presto Version: presto-main:0.97-SNAPSHOT 5 | Node status: active 6 | Catalogs: hive, system, tpch 7 | Server Status: 8 | Node2(IP: IP2, Roles: worker): Running 9 | Node URI(http): http://inactive/stmt 10 | Presto Version: presto-main:0.99-SNAPSHOT 11 | Node status: inactive 12 | Catalogs: hive, system, tpch 13 | Server Status: 14 | Node3(IP: IP3, Roles: worker): Running 15 | No information available: the coordinator has not yet discovered this node 16 | Server Status: 17 | Node4(IP: Unknown, Roles: worker): Not Running 18 | Timed out trying to connect to Node4 19 | -------------------------------------------------------------------------------- /tests/unit/resources/slider-extended-help.txt: -------------------------------------------------------------------------------- 1 | Usage: presto-admin [options] [arg] 2 | 3 | Options: 4 | --version show program's version number and exit 5 | -h, --help show this help message and exit 6 | -d, --display print detailed information about command 7 | --extended-help print out all options, including advanced ones 8 | -I, --initial-password-prompt 9 | Force password prompt up-front 10 | -p PASSWORD, --password=PASSWORD 11 | password for use with authentication and/or sudo 12 | 13 | Advanced Options: 14 | -a, --no_agent don't use the running SSH agent 15 | -A, --forward-agent 16 | forward local agent to remote end 17 | --colorize-errors Color error output 18 | -D, --disable-known-hosts 19 | do not load user known_hosts file 20 | -g HOST, --gateway=HOST 21 | gateway host to connect through 22 | -H HOSTS, --hosts=HOSTS 23 | comma-separated list of hosts to operate on 24 | -i PATH path to SSH private key file. May be repeated. 25 | -k, --no-keys don't load private key files from ~/.ssh/ 26 | --keepalive=N enables a keepalive every N seconds 27 | -n M, --connection-attempts=M 28 | make M attempts to connect before giving up 29 | --port=PORT SSH connection port 30 | -r, --reject-unknown-hosts 31 | reject unknown hosts 32 | --system-known-hosts=SYSTEM_KNOWN_HOSTS 33 | load system known_hosts file before reading user 34 | known_hosts 35 | -t N, --timeout=N set connection timeout to N seconds 36 | -T N, --command-timeout=N 37 | set remote command timeout to N seconds 38 | -u USER, --user=USER 39 | username to use when connecting to remote hosts 40 | -x HOSTS, --exclude-hosts=HOSTS 41 | comma-separated list of hosts to exclude 42 | --serial default to serial execution method 43 | 44 | Commands: 45 | server install 46 | server uninstall 47 | slider install 48 | slider uninstall 49 | 50 | -------------------------------------------------------------------------------- /tests/unit/resources/slider-help.txt: -------------------------------------------------------------------------------- 1 | Usage: presto-admin [options] [arg] 2 | 3 | Options: 4 | --version show program's version number and exit 5 | -h, --help show this help message and exit 6 | -d, --display print detailed information about command 7 | --extended-help print out all options, including advanced ones 8 | -I, --initial-password-prompt 9 | Force password prompt up-front 10 | -p PASSWORD, --password=PASSWORD 11 | password for use with authentication and/or sudo 12 | 13 | 14 | Commands: 15 | server install 16 | server uninstall 17 | slider install 18 | slider uninstall 19 | 20 | -------------------------------------------------------------------------------- /tests/unit/resources/standalone-extended-help.txt: -------------------------------------------------------------------------------- 1 | Usage: presto-admin [options] [arg] 2 | 3 | Options: 4 | --version show program's version number and exit 5 | -h, --help show this help message and exit 6 | -d, --display print detailed information about command 7 | --extended-help print out all options, including advanced ones 8 | -I, --initial-password-prompt 9 | Force password prompt up-front 10 | -p PASSWORD, --password=PASSWORD 11 | password for use with authentication and/or sudo 12 | 13 | Advanced Options: 14 | -a, --no_agent don't use the running SSH agent 15 | -A, --forward-agent 16 | forward local agent to remote end 17 | --colorize-errors Color error output 18 | -D, --disable-known-hosts 19 | do not load user known_hosts file 20 | -g HOST, --gateway=HOST 21 | gateway host to connect through 22 | -H HOSTS, --hosts=HOSTS 23 | comma-separated list of hosts to operate on 24 | -i PATH path to SSH private key file. May be repeated. 25 | -k, --no-keys don't load private key files from ~/.ssh/ 26 | --keepalive=N enables a keepalive every N seconds 27 | -n M, --connection-attempts=M 28 | make M attempts to connect before giving up 29 | --port=PORT SSH connection port 30 | -r, --reject-unknown-hosts 31 | reject unknown hosts 32 | --system-known-hosts=SYSTEM_KNOWN_HOSTS 33 | load system known_hosts file before reading user 34 | known_hosts 35 | -t N, --timeout=N set connection timeout to N seconds 36 | -T N, --command-timeout=N 37 | set remote command timeout to N seconds 38 | -u USER, --user=USER 39 | username to use when connecting to remote hosts 40 | -x HOSTS, --exclude-hosts=HOSTS 41 | comma-separated list of hosts to exclude 42 | --serial default to serial execution method 43 | 44 | Commands: 45 | catalog add 46 | catalog remove 47 | collect logs 48 | collect query_info 49 | collect system_info 50 | configuration deploy 51 | configuration show 52 | file copy 53 | file run 54 | package install 55 | package uninstall 56 | plugin add_jar 57 | server install 58 | server restart 59 | server start 60 | server status 61 | server stop 62 | server uninstall 63 | server upgrade 64 | topology show 65 | 66 | -------------------------------------------------------------------------------- /tests/unit/resources/standalone-help.txt: -------------------------------------------------------------------------------- 1 | Usage: presto-admin [options] [arg] 2 | 3 | Options: 4 | --version show program's version number and exit 5 | -h, --help show this help message and exit 6 | -d, --display print detailed information about command 7 | --extended-help print out all options, including advanced ones 8 | -I, --initial-password-prompt 9 | Force password prompt up-front 10 | -p PASSWORD, --password=PASSWORD 11 | password for use with authentication and/or sudo 12 | 13 | 14 | Commands: 15 | catalog add 16 | catalog remove 17 | collect logs 18 | collect query_info 19 | collect system_info 20 | configuration deploy 21 | configuration show 22 | file copy 23 | file run 24 | package install 25 | package uninstall 26 | plugin add_jar 27 | server install 28 | server restart 29 | server start 30 | server status 31 | server stop 32 | server uninstall 33 | server upgrade 34 | topology show 35 | 36 | -------------------------------------------------------------------------------- /tests/unit/resources/valid.config: -------------------------------------------------------------------------------- 1 | prop1 2 | prop2 3 | prop3 -------------------------------------------------------------------------------- /tests/unit/resources/valid.properties: -------------------------------------------------------------------------------- 1 | a=1 2 | b:2 3 | c 3 4 | ! A comment 5 | # another comment 6 | d\== 4 7 | e\:=5 8 | f===6 9 | g:= 7 10 | h=:8 11 | i = 9 12 | 13 | -------------------------------------------------------------------------------- /tests/unit/resources/valid_rest_response_level1.txt: -------------------------------------------------------------------------------- 1 | {"id":"2015_harih","infoUri":"http://localhost:8080/v1/query/2015_harih","nextUri":"http://localhost:8080/v1/statement/2015_harih/2"} -------------------------------------------------------------------------------- /tests/unit/resources/valid_rest_response_level2.txt: -------------------------------------------------------------------------------- 1 | {"id":"2015_harih","infoUri":"http://localhost:8080/v1/query/2015_harih","nextUri":"","data":[["uuid1","http://localhost:8080","presto-main:0.97",true], ["uuid2","http://worker:8080","presto-main:0.97",false]]} -------------------------------------------------------------------------------- /tests/unit/standalone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/unit/standalone/__init__.py -------------------------------------------------------------------------------- /tests/unit/standalone/test_help.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from mock import patch 17 | import os 18 | 19 | import prestoadmin 20 | from prestoadmin import main 21 | 22 | from tests.unit.test_main import BaseMainCase 23 | 24 | 25 | # Consult the comment on yarn_slider.test_help.TestSliderHelp for more info. 26 | class TestStandaloneHelp(BaseMainCase): 27 | @patch('prestoadmin.mode.get_mode', return_value='standalone') 28 | def setUp(self, mode_mock): 29 | super(TestStandaloneHelp, self).setUp() 30 | reload(prestoadmin) 31 | reload(main) 32 | 33 | def get_short_help_path(self): 34 | return os.path.join('resources', 'standalone-help.txt') 35 | 36 | def get_extended_help_path(self): 37 | return os.path.join('resources', 'standalone-extended-help.txt') 38 | 39 | def test_standalone_help_text_short(self): 40 | self._run_command_compare_to_file( 41 | ["-h"], 0, self.get_short_help_path()) 42 | 43 | def test_standalone_help_text_long(self): 44 | self._run_command_compare_to_file( 45 | ["--help"], 0, self.get_short_help_path()) 46 | 47 | def test_standalone_help_displayed_with_no_args(self): 48 | self._run_command_compare_to_file( 49 | [], 0, self.get_short_help_path()) 50 | 51 | def test_standalone_extended_help(self): 52 | self._run_command_compare_to_file( 53 | ['--extended-help'], 0, self.get_extended_help_path()) 54 | -------------------------------------------------------------------------------- /tests/unit/test_base_test_case.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Module for validating functionality in BaseTestCase. 17 | """ 18 | 19 | from base_unit_case import BaseUnitCase 20 | 21 | 22 | class TestBaseTestCase(BaseUnitCase): 23 | def testLazyPass(self): 24 | self.assertLazyMessage( 25 | lambda: self.fail("shouldn't be called"), self.assertEqual, 1, 1) 26 | 27 | def testLazyFail(self): 28 | a = 2 29 | e = 1 30 | 31 | self.assertRaisesRegexp( 32 | AssertionError, 'asdfasdfasdf 2 1', self.assertLazyMessage, 33 | lambda: 'asdfasdfasdf %d %d' % (a, e), self.assertEqual, a, e) 34 | -------------------------------------------------------------------------------- /tests/unit/test_expand.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from prestoadmin.standalone.config import _expand_host 16 | from tests.unit.base_unit_case import BaseUnitCase 17 | 18 | 19 | class TestExpandHost(BaseUnitCase): 20 | 21 | def test_basic_expand_host_01(self): 22 | input_host = "worker0[1-2].example.com" 23 | expected = ["worker01.example.com", "worker02.example.com"] 24 | self.assertEqual(expected, _expand_host(input_host)) 25 | 26 | def test_basic_expand_host_02(self): 27 | input_host = "worker[01-02].example.com" 28 | expected = ["worker01.example.com", "worker02.example.com"] 29 | self.assertEqual(expected, _expand_host(input_host)) 30 | 31 | def test_expand_host_include_hyphen(self): 32 | input_host = "cdh5-[1-2].example.com" 33 | expected = ["cdh5-1.example.com", "cdh5-2.example.com"] 34 | self.assertEqual(expected, _expand_host(input_host)) 35 | 36 | def test_not_expand_host(self): 37 | input_host = "worker1.example.com" 38 | expected = ["worker1.example.com"] 39 | self.assertEqual(expected, _expand_host(input_host)) 40 | 41 | def test_except_expand_host(self): 42 | input_host = "worker0[3-2].example.com" 43 | self.assertRaises(ValueError, _expand_host, input_host) 44 | -------------------------------------------------------------------------------- /tests/unit/test_file.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Tests the script module 17 | """ 18 | 19 | from mock import patch, call 20 | from prestoadmin import file 21 | 22 | from tests.unit.base_unit_case import BaseUnitCase 23 | 24 | 25 | class TestFile(BaseUnitCase): 26 | 27 | @patch('prestoadmin.file.sudo') 28 | @patch('prestoadmin.file.put') 29 | def test_script_basic(self, put_mock, sudo_mock): 30 | file.run('/my/local/path/script.sh') 31 | put_mock.assert_called_with('/my/local/path/script.sh', 32 | '/tmp/script.sh') 33 | sudo_mock.assert_has_calls( 34 | [call('chmod u+x /tmp/script.sh'), call('/tmp/script.sh'), 35 | call('rm /tmp/script.sh')], any_order=False) 36 | 37 | @patch('prestoadmin.file.sudo') 38 | @patch('prestoadmin.file.put') 39 | def test_script_specify_dir(self, put_mock, sudo_mock): 40 | file.run('/my/local/path/script.sh', '/my/remote/path') 41 | put_mock.assert_called_with('/my/local/path/script.sh', 42 | '/my/remote/path/script.sh') 43 | sudo_mock.assert_has_calls( 44 | [call('chmod u+x /my/remote/path/script.sh'), 45 | call('/my/remote/path/script.sh'), 46 | call('rm /my/remote/path/script.sh')], any_order=False) 47 | -------------------------------------------------------------------------------- /tests/unit/test_plugin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | unit tests for plugin module 17 | """ 18 | from mock import patch 19 | from prestoadmin import plugin 20 | from tests.unit.base_unit_case import BaseUnitCase 21 | 22 | 23 | class TestPlugin(BaseUnitCase): 24 | @patch('prestoadmin.plugin.write') 25 | def test_add_jar(self, write_mock): 26 | plugin.add_jar('/my/local/path.jar', 'hive-hadoop2') 27 | write_mock.assert_called_with( 28 | '/my/local/path.jar', '/usr/lib/presto/lib/plugin/hive-hadoop2') 29 | 30 | @patch('prestoadmin.plugin.write') 31 | def test_add_jar_provide_dir(self, write_mock): 32 | plugin.add_jar('/my/local/path.jar', 'hive-hadoop2', 33 | '/etc/presto/plugin') 34 | write_mock.assert_called_with('/my/local/path.jar', 35 | '/etc/presto/plugin/hive-hadoop2') 36 | -------------------------------------------------------------------------------- /tests/unit/test_presto_config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from StringIO import StringIO 15 | 16 | from prestoadmin.util.presto_config import PrestoConfig 17 | from tests.unit.base_unit_case import BaseUnitCase 18 | 19 | 20 | class TestPrestoConfig(BaseUnitCase): 21 | realworld = """ 22 | coordinator=true 23 | discovery-server.enabled=true 24 | discovery.uri=http://localhost:8285 25 | http-server.http.port=8285 26 | node-scheduler.include-coordinator=true 27 | query.max-memory-per-node=8GB 28 | query.max-memory=50GB 29 | http-server.https.port=8444 30 | http-server.https.enabled=true 31 | http-server.https.keystore.path=/tmp/mykeystore.jks 32 | http-server.https.keystore.key=testldap 33 | http-server.authentication.type=LDAP 34 | authentication.ldap.url=ldaps://10.25.171.180:636 35 | authentication.ldap.user-bind-pattern=${USER}@presto.testldap.com 36 | """ 37 | 38 | def _get_presto_config(self, config): 39 | config_file = StringIO(config) 40 | return PrestoConfig.from_file(config_file) 41 | 42 | def _assert_use_https(self, expected, config): 43 | presto_config = self._get_presto_config(config) 44 | self.assertEqual(presto_config.use_https(), expected) 45 | 46 | def test_use_https(self): 47 | self._assert_use_https(False, "") 48 | self._assert_use_https(False, "http-server.http.enabled=true") 49 | self._assert_use_https(False, "http-server.https.enabled=true") 50 | 51 | self._assert_use_https(False, """ 52 | http-server.http.enabled=true 53 | http-server.https.enabled=true") 54 | """) 55 | 56 | self._assert_use_https(True, """ 57 | http-server.http.enabled=false 58 | http-server.https.enabled=true 59 | """) 60 | 61 | self._assert_use_https(False, self.realworld) 62 | 63 | def _assert_use_ldap(self, expected, config): 64 | presto_config = self._get_presto_config(config) 65 | self.assertEqual(presto_config.use_ldap(), expected) 66 | 67 | def test_use_ldap(self): 68 | self._assert_use_ldap(False, "") 69 | self._assert_use_ldap(False, "http-server.authentication.type=LDAP") 70 | 71 | self._assert_use_ldap(False, """ 72 | http-server.http.enabled=false 73 | http-server.https.enabled=true 74 | http-server.authentication.type=A_BIG_BRASS_KEY 75 | """) 76 | 77 | self._assert_use_ldap(True, """ 78 | http-server.http.enabled=false 79 | http-server.https.enabled=true 80 | http-server.authentication.type=LDAP 81 | """) 82 | 83 | self._assert_use_ldap(False, self.realworld) 84 | -------------------------------------------------------------------------------- /tests/unit/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/unit/util/__init__.py -------------------------------------------------------------------------------- /tests/unit/util/test_exception.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from prestoadmin.util.exception import ExceptionWithCause, \ 16 | ConfigFileNotFoundError 17 | 18 | import pickle 19 | import re 20 | from unittest import TestCase 21 | 22 | 23 | class ExceptionTest(TestCase): 24 | 25 | def test_exception_with_cause(self): 26 | pass 27 | try: 28 | try: 29 | raise ValueError('invalid parameter!') 30 | except: 31 | raise ExceptionWithCause('outer exception') 32 | except ExceptionWithCause as e: 33 | self.assertEqual(str(e), 'outer exception') 34 | m = re.match( 35 | r'Traceback \(most recent call last\):\n File ".*", line \d+,' 36 | ' in test_exception_with_cause\n raise ValueError\(' 37 | '\'invalid parameter!\'\)\nValueError: invalid parameter!\n', 38 | e.inner_exception 39 | ) 40 | self.assertTrue(m is not None) 41 | else: 42 | self.fail('ExceptionWithCause should have been raised') 43 | 44 | def test_can_pickle_ConfigFileNotFound(self): 45 | config_path = '/usa/georgia/macon' 46 | message = 'I woke up this morning, I had them Statesboro Blues' 47 | e = ConfigFileNotFoundError(config_path=config_path, message=message) 48 | 49 | ps = pickle.dumps(e, pickle.HIGHEST_PROTOCOL) 50 | a = pickle.loads(ps) 51 | 52 | self.assertEquals(message, a.message) 53 | self.assertEquals(config_path, a.config_path) 54 | -------------------------------------------------------------------------------- /tests/unit/util/test_fabric_application.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from prestoadmin.util.fabric_application import FabricApplication 16 | 17 | from tests.base_test_case import BaseTestCase 18 | 19 | from mock import patch 20 | 21 | import sys 22 | import logging 23 | 24 | 25 | APPLICATION_NAME = 'foo' 26 | 27 | 28 | @patch('prestoadmin.util.application.logging.config') 29 | class FabricApplicationTest(BaseTestCase): 30 | 31 | def setUp(self): 32 | # basicConfig is a noop if there are already handlers 33 | # present on the root logger, remove them all here 34 | self.__old_log_handlers = [] 35 | for handler in logging.root.handlers: 36 | self.__old_log_handlers.append(handler) 37 | logging.root.removeHandler(handler) 38 | super(FabricApplicationTest, self).setUp(capture_output=True) 39 | 40 | def tearDown(self): 41 | # restore the old log handlers 42 | for handler in logging.root.handlers: 43 | logging.root.removeHandler(handler) 44 | for handler in self.__old_log_handlers: 45 | logging.root.addHandler(handler) 46 | BaseTestCase.tearDown(self) 47 | 48 | @patch('prestoadmin.util.fabric_application.disconnect_all', autospec=True) 49 | def test_disconnect_all(self, disconnect_mock, logging_conf_mock): 50 | def should_disconnect(): 51 | with FabricApplication(APPLICATION_NAME): 52 | sys.exit() 53 | 54 | self.assertRaises(SystemExit, should_disconnect) 55 | disconnect_mock.assert_called_with() 56 | 57 | @patch('prestoadmin.util.application.logger') 58 | @patch('prestoadmin.util.filesystem.os.makedirs') 59 | def test_keyboard_interrupt(self, make_dirs_mock, logger_mock, 60 | logging_conf_mock): 61 | def should_not_error(): 62 | with FabricApplication(APPLICATION_NAME): 63 | raise KeyboardInterrupt 64 | 65 | try: 66 | should_not_error() 67 | except SystemExit as e: 68 | self.assertEqual(0, e.code) 69 | self.assertEqual("Stopped.\n", self.test_stderr.getvalue()) 70 | else: 71 | self.fail('Keyboard interrupt did not cause a system exit.') 72 | 73 | def test_handles_errors(self, logging_mock): 74 | def should_fail(): 75 | with FabricApplication(APPLICATION_NAME): 76 | raise Exception('error message') 77 | 78 | self.assertRaises(SystemExit, should_fail) 79 | self.assertEqual(self.test_stderr.getvalue(), 'error message\n') 80 | -------------------------------------------------------------------------------- /tests/unit/util/test_fabricapi.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Tests the utility 17 | """ 18 | from fabric.api import env 19 | 20 | from mock import Mock 21 | 22 | from prestoadmin.util import fabricapi 23 | 24 | from tests.base_test_case import BaseTestCase 25 | 26 | 27 | class TestFabricapi(BaseTestCase): 28 | def test_get_host_with_exclude(self): 29 | env.hosts = ['a', 'b', 'bad'] 30 | env.exclude_hosts = ['bad'] 31 | self.assertEqual(fabricapi.get_host_list(), ['a', 'b']) 32 | 33 | TEST_ROLEDEFS = { 34 | 'coordinator': ['coordinator'], 35 | 'worker': ['worker0', 'worker1', 'worker2'] 36 | } 37 | 38 | def test_by_role_coordinator(self): 39 | env.roledefs = self.TEST_ROLEDEFS 40 | 41 | callback = Mock() 42 | 43 | fabricapi.by_role_coordinator('worker0', callback) 44 | self.assertFalse(callback.called, 'coordinator callback called for ' + 45 | 'worker') 46 | fabricapi.by_role_coordinator('coordinator', callback) 47 | callback.assert_any_call() 48 | 49 | def test_by_role_worker(self): 50 | env.roledefs = self.TEST_ROLEDEFS 51 | 52 | callback = Mock() 53 | 54 | fabricapi.by_role_worker('coordinator', callback) 55 | self.assertFalse(callback.called, 'worker callback called for ' + 56 | 'coordinator') 57 | fabricapi.by_role_worker('worker0', callback) 58 | callback.assert_any_call() 59 | 60 | def assert_is_worker(self, roledefs): 61 | def check(*args, **kwargs): 62 | self.assertTrue(env.host in roledefs.get('worker')) 63 | return check 64 | 65 | def assert_is_coordinator(self, roledefs): 66 | def check(*args, **kwargs): 67 | self.assertTrue(env.host in roledefs.get('coordinator')) 68 | return check 69 | 70 | def test_by_rolename_worker(self): 71 | callback = Mock() 72 | callback.side_effect = self.assert_is_worker(self.TEST_ROLEDEFS) 73 | env.roledefs = self.TEST_ROLEDEFS 74 | 75 | env.host = 'coordinator' 76 | fabricapi.by_rolename(env.host, 'worker', callback) 77 | self.assertFalse(callback.called) 78 | 79 | env.host = 'worker0' 80 | fabricapi.by_rolename(env.host, 'worker', callback) 81 | self.assertTrue(callback.called) 82 | 83 | def test_by_rolename_coordinator(self): 84 | callback = Mock() 85 | callback.side_effect = self.assert_is_coordinator(self.TEST_ROLEDEFS) 86 | env.roledefs = self.TEST_ROLEDEFS 87 | 88 | env.host = 'worker0' 89 | fabricapi.by_rolename(env.host, 'coordinator', callback) 90 | self.assertFalse(callback.called) 91 | 92 | env.host = 'coordinator' 93 | fabricapi.by_rolename(env.host, 'coordinator', callback) 94 | self.assertTrue(callback.called) 95 | 96 | def test_by_rolename_all(self): 97 | callback = Mock() 98 | env.roledefs = self.TEST_ROLEDEFS 99 | 100 | env.host = 'worker0' 101 | fabricapi.by_rolename(env.host, None, callback) 102 | self.assertTrue(callback.called) 103 | 104 | callback.reset_mock() 105 | 106 | env.host = 'coordinator' 107 | fabricapi.by_rolename(env.host, None, callback) 108 | self.assertTrue(callback.called) 109 | -------------------------------------------------------------------------------- /tests/unit/util/test_filesystem.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import errno 16 | from mock import patch 17 | from prestoadmin.util import filesystem 18 | from tests.base_test_case import BaseTestCase 19 | 20 | 21 | class TestFilesystem(BaseTestCase): 22 | @patch('prestoadmin.util.filesystem.os.fdopen') 23 | @patch('prestoadmin.util.filesystem.os.open') 24 | @patch('prestoadmin.util.filesystem.os.makedirs') 25 | def test_write_file_exits(self, makedirs_mock, open_mock, fdopen_mock): 26 | makedirs_mock.side_effect = OSError(errno.EEXIST, 'message') 27 | open_mock.side_effect = OSError(errno.EEXIST, 'message') 28 | filesystem.write_to_file_if_not_exists('content', 'path/to/anyfile') 29 | self.assertFalse(fdopen_mock.called) 30 | 31 | @patch('prestoadmin.util.filesystem.os.makedirs') 32 | def test_write_file_error_in_dirs(self, makedirs_mock): 33 | makedirs_mock.side_effect = OSError(errno.EACCES, 'message') 34 | self.assertRaisesRegexp(OSError, 'message', 35 | filesystem.write_to_file_if_not_exists, 36 | 'content', 'path/to/anyfile') 37 | 38 | @patch('prestoadmin.util.filesystem.os.makedirs') 39 | @patch('prestoadmin.util.filesystem.os.open') 40 | def test_write_file_error_in_files(self, open_mock, makedirs_mock): 41 | open_mock.side_effect = OSError(errno.EACCES, 'message') 42 | self.assertRaisesRegexp(OSError, 'message', 43 | filesystem.write_to_file_if_not_exists, 44 | 'content', 'path/to/anyfile') 45 | -------------------------------------------------------------------------------- /tests/unit/util/test_local_config_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import os 16 | 17 | from mock import patch 18 | from prestoadmin.util import local_config_util 19 | from prestoadmin.util.constants import DEFAULT_LOCAL_CONF_DIR 20 | from tests.base_test_case import BaseTestCase 21 | 22 | 23 | class TestLocalConfigUtil(BaseTestCase): 24 | @patch('prestoadmin.util.local_config_util.os.environ.get') 25 | def test_get_default_config_dir(self, get_mock): 26 | get_mock.return_value = None 27 | self.assertEqual(local_config_util.get_config_directory(), DEFAULT_LOCAL_CONF_DIR) 28 | 29 | @patch('prestoadmin.util.local_config_util.os.environ.get') 30 | def test_get_configured_config_dir(self, get_mock): 31 | non_default_directory = '/not/the/default' 32 | get_mock.return_value = non_default_directory 33 | self.assertEqual(local_config_util.get_config_directory(), non_default_directory) 34 | 35 | @patch('prestoadmin.util.local_config_util.os.environ.get') 36 | def test_get_default_log_dir(self, get_mock): 37 | get_mock.return_value = None 38 | self.assertEqual(local_config_util.get_log_directory(), os.path.join(DEFAULT_LOCAL_CONF_DIR, 'log')) 39 | 40 | @patch('prestoadmin.util.local_config_util.os.environ.get') 41 | def test_get_configured_log_dir(self, get_mock): 42 | non_default_directory = '/not/the/default' 43 | get_mock.return_value = non_default_directory 44 | self.assertEqual(local_config_util.get_log_directory(), non_default_directory) 45 | -------------------------------------------------------------------------------- /tests/unit/util/test_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Tests the LoggingOptionParser 17 | """ 18 | from StringIO import StringIO 19 | 20 | from prestoadmin.util.parser import LoggingOptionParser 21 | from prestoadmin.util.hiddenoptgroup import HiddenOptionGroup 22 | from tests.base_test_case import BaseTestCase 23 | 24 | 25 | class TestParser(BaseTestCase): 26 | def test_print_extended_help(self): 27 | parser = LoggingOptionParser(usage="Hello World") 28 | parser.add_option_group("a") 29 | hidden_group = HiddenOptionGroup(parser, "b", suppress_help=True) 30 | non_hidden_group = HiddenOptionGroup(parser, "c", suppress_help=False) 31 | parser.add_option_group(hidden_group) 32 | parser.add_option_group(non_hidden_group) 33 | 34 | help_out = StringIO() 35 | parser.print_help(help_out) 36 | self.assertEqual(help_out.getvalue(), 37 | "Usage: Hello World\n\nOptions:\n -h, --help show " 38 | "this help message and exit\n\n a:\n\n\n c:\n") 39 | 40 | extended_help_out = StringIO() 41 | parser.print_extended_help(extended_help_out) 42 | self.assertEqual(extended_help_out.getvalue(), 43 | "Usage: Hello World\n\nOptions:\n -h, --help show " 44 | "this help message and exit\n\n a:\n\n b:\n\n " 45 | "c:\n") 46 | -------------------------------------------------------------------------------- /tests/unit/util/test_validators.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Test the various validators 17 | """ 18 | from prestoadmin.util import validators 19 | from prestoadmin.util.exception import ConfigurationError 20 | from tests.base_test_case import BaseTestCase 21 | 22 | 23 | class TestValidators(BaseTestCase): 24 | def test_valid_ipv4(self): 25 | ipv4 = "10.14.1.10" 26 | self.assertEqual(validators.validate_host(ipv4), ipv4) 27 | 28 | def test_valid_full_ipv6(self): 29 | ipv6 = "FE80:0000:0000:0000:0202:B3FF:FE1E:8329" 30 | self.assertEqual(validators.validate_host(ipv6), ipv6) 31 | 32 | def test_valid_collapsed_ipv6(self): 33 | ipv6 = "FE80::0202:B3FF:FE1E:8329" 34 | self.assertEqual(validators.validate_host(ipv6), ipv6) 35 | 36 | def test_valid_hostname(self): 37 | host = "master" 38 | self.assertEqual(validators.validate_host(host), host) 39 | 40 | def test_invalid_host(self): 41 | self.assertRaisesRegexp(ConfigurationError, 42 | "'.1234' is not a valid ip address " 43 | "or host name", 44 | validators.validate_host, 45 | (".1234")) 46 | 47 | def test_invalid_host_type(self): 48 | self.assertRaisesRegexp(ConfigurationError, 49 | "Host must be of type string. " 50 | "Found ", 51 | validators.validate_host, 52 | (["my", "list"])) 53 | 54 | def test_valid_port(self): 55 | port = 1234 56 | self.assertEqual(validators.validate_port(port), port) 57 | 58 | def test_invalid_port(self): 59 | self.assertRaisesRegexp(ConfigurationError, 60 | "Invalid port number 99999999: port must be " 61 | "a number between 1 and 65535", 62 | validators.validate_port, 63 | ("99999999")) 64 | 65 | def test_invalid_port_type(self): 66 | self.assertRaises(ConfigurationError, 67 | validators.validate_port, (["123"])) 68 | -------------------------------------------------------------------------------- /tests/unit/yarn_slider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/presto-admin/6e7aee3427bdbea6da2deb41b7f090ef6fdcadd9/tests/unit/yarn_slider/__init__.py -------------------------------------------------------------------------------- /tests/unit/yarn_slider/test_help.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from mock import patch 17 | import os 18 | 19 | import prestoadmin 20 | from prestoadmin import main 21 | 22 | from tests.unit.test_main import BaseMainCase 23 | 24 | 25 | # 26 | # Getting this and TestStandaloneHelp running, and subsequently running 27 | # successfully in the same run was a treat and a joy not to be missed. 28 | # 29 | # A 30 | # Because the import runs way up there |, and __init__.py runs get_mode, it's 31 | # basically impossible to patch get_mode using the usual mechanisms; the mode 32 | # has long been set by the time we get to setUp or any of the tests. Instead, 33 | # we patch it down here, and then reload the prestoadmin module to re-execute 34 | # the code that calls get_mode and sets up the imports and __all__. 35 | # 36 | # The other thing to keep in mind is that the help tests end up (many levels 37 | # in) updating fabric.state.commands, and you need to clear it out in order for 38 | # the second test case to run correctly. BaseMainCase.setUp does this because 39 | # TestMain also ends up updating fabric.state.commands, and therefore ought to 40 | # clear it too. 41 | # 42 | # There's a lot of duplication between this and TestStandaloneHelp. Here are a 43 | # few things that don't work to remove it: 44 | # 45 | # Have a common abstract base class. Nosetests tries to instantiate it. 46 | # Mark the base class @nottest. Nosetests doesn't find the tests in the 47 | # concrete classes. 48 | # Common non-abstract base class with additional constructor args. Nosetest 49 | # will probably try to instantiate that too. 50 | # Multiple inheritance. Now you have two problems ;-) 51 | # 52 | class TestSliderHelp(BaseMainCase): 53 | @patch('prestoadmin.mode.get_mode', return_value='yarn_slider') 54 | def setUp(self, mode_mock): 55 | super(TestSliderHelp, self).setUp() 56 | reload(prestoadmin) 57 | reload(main) 58 | 59 | def get_short_help_path(self): 60 | return os.path.join('resources', 'slider-help.txt') 61 | 62 | def get_extended_help_path(self): 63 | return os.path.join('resources', 'slider-extended-help.txt') 64 | 65 | def test_standalone_help_text_short(self): 66 | self._run_command_compare_to_file( 67 | ["-h"], 0, self.get_short_help_path()) 68 | 69 | def test_standalone_help_text_long(self): 70 | self._run_command_compare_to_file( 71 | ["--help"], 0, self.get_short_help_path()) 72 | 73 | def test_standalone_help_displayed_with_no_args(self): 74 | self._run_command_compare_to_file( 75 | [], 0, self.get_short_help_path()) 76 | 77 | def test_standalone_extended_help(self): 78 | self._run_command_compare_to_file( 79 | ['--extended-help'], 0, self.get_extended_help_path()) 80 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26, py27 3 | 4 | [testenv] 5 | setenv = 6 | PYTHONPATH = {toxinidir}:{toxinidir}/prestoadmin 7 | commands = nosetests --with-timer --timer-ok 60s --timer-warning 300s {posargs} 8 | deps = 9 | -r{toxinidir}/requirements.txt 10 | passenv = DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH 11 | -------------------------------------------------------------------------------- /util/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Modules within util should only use the standard library because setup.py 16 | # may rely on the modules. setup.py typically installs all dependencies. 17 | # If a third party module is used, setup.py may attempt to import it while 18 | # trying to install dependencies and an ImportError will be raised because 19 | # the dependency has not been installed yet. 20 | import os 21 | 22 | main_dir = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) 23 | 24 | with open(os.path.join(main_dir, 'prestoadmin/_version.py')) as version_file: 25 | __version__ = version_file.readlines()[-1].split()[-1].strip("\"'") 26 | -------------------------------------------------------------------------------- /util/http.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Module for sending HTTP requests 16 | """ 17 | import urllib2 18 | 19 | 20 | def send_get_request(url): 21 | response = None 22 | try: 23 | response = urllib2.urlopen(url) 24 | if response.getcode() != 200: 25 | exit('Get request to %s responded with status of %s' % (url, str(response.getcode()))) 26 | else: 27 | headers = response.info() 28 | contents = response.read() 29 | return headers, contents 30 | finally: 31 | if response: 32 | response.close() 33 | 34 | 35 | def send_authorized_post_request(url, data, authorization_string, content_type, content_length): 36 | response = None 37 | try: 38 | request = urllib2.Request(url, data, 39 | {'Content-Type': '%s' % content_type, 40 | 'Content-Length': content_length, 41 | 'Authorization': 'Basic %s' % authorization_string}) 42 | response = urllib2.urlopen(request) 43 | status = response.getcode() 44 | headers = response.info() 45 | contents = response.read() 46 | if status != 201: 47 | print headers 48 | print contents 49 | exit('Failed to post to %s' % url) 50 | finally: 51 | if response: 52 | response.close() 53 | -------------------------------------------------------------------------------- /util/semantic_version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Module for parsing and processing semantic versions 16 | """ 17 | class SemanticVersion(object): 18 | def __init__(self, version): 19 | self.version = version 20 | version_fields = self.version.split('.') 21 | if len(version_fields) > 3: 22 | exit('Version %s has more than 3 fields' % self.version) 23 | self.major_version = self._get_version_field_value(version_fields, 0) 24 | self.minor_version = self._get_version_field_value(version_fields, 1) 25 | self.patch_version = self._get_version_field_value(version_fields, 2) 26 | 27 | def _get_version_field_value(self, version_fields, index): 28 | try: 29 | return int(version_fields[index]) 30 | except IndexError: 31 | # The field value was omitted for the version 32 | return 0 33 | except ValueError: 34 | exit('Version %s has a non-numeric field' % self.version) 35 | 36 | def __lt__(self, other): 37 | if self.major_version == other.major_version: 38 | if self.minor_version == other.minor_version: 39 | return self.patch_version < other.patch_version 40 | else: 41 | return self.minor_version < other.minor_version 42 | else: 43 | return self.major_version < other.major_version 44 | 45 | def __eq__(self, other): 46 | return self.major_version == other.major_version and \ 47 | self.minor_version == other.minor_version and \ 48 | self.patch_version == other.patch_veresion 49 | 50 | def __str__(self): 51 | return self.version 52 | 53 | @staticmethod 54 | def _bump_version(version_field): 55 | return str(int(version_field) + 1) 56 | 57 | def _get_acceptable_major_version_bumps(self): 58 | acceptable_major = self._bump_version(self.major_version) 59 | return [acceptable_major, 60 | acceptable_major + '.0', 61 | acceptable_major + '.0.0'] 62 | 63 | def _get_acceptable_minor_version_bumps(self): 64 | acceptable_minor = self._bump_version(self.minor_version) 65 | return [str(self.major_version) + '.' + acceptable_minor, 66 | str(self.major_version) + '.' + acceptable_minor + '.0'] 67 | 68 | def _get_acceptable_patch_version_bumps(self): 69 | acceptable_patch = self._bump_version(self.patch_version) 70 | return [str(self.major_version) + '.' + str(self.minor_version) + '.' + acceptable_patch] 71 | 72 | def get_acceptable_version_bumps(self): 73 | """ 74 | This functions takes as input strings major, minor, and patch which should be 75 | the corresponding semvar fields for a release. It returns a list of strings, which 76 | contains all acceptable versions. For each field bump, lower fields may be omitted 77 | or 0s. For instance, bumping 0.1.2's major version can result in 1, 1.0, or 1.0.0. 78 | """ 79 | major_bumps = self._get_acceptable_major_version_bumps() 80 | minor_bumps = self._get_acceptable_minor_version_bumps() 81 | patch_bumps = self._get_acceptable_patch_version_bumps() 82 | return major_bumps + minor_bumps + patch_bumps 83 | --------------------------------------------------------------------------------