├── .gitignore ├── .travis.yml ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── configuration ├── config.properties.xml ├── connectors.properties.xml ├── jvm.config.xml └── node.properties.xml ├── docs ├── Makefile ├── conf.py ├── developers.rst ├── getting-help.rst ├── getting-started.rst ├── index.rst └── known-issues.rst ├── metainfo.xml ├── package ├── __init__.py └── scripts │ ├── __init__.py │ ├── common.py │ ├── download.ini │ ├── params.py │ ├── presto_cli.py │ ├── presto_client.py │ ├── presto_coordinator.py │ └── presto_worker.py ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── resource_management │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── exceptions.py │ │ └── resources │ │ │ ├── __init__.py │ │ │ └── system.py │ └── libraries │ │ ├── __init__.py │ │ └── script │ │ ├── __init__.py │ │ └── script.py ├── resources │ ├── valid_rest_response_level1.txt │ └── valid_rest_response_level2.txt ├── test_cli.py ├── test_common.py ├── test_coordinator.py ├── test_presto_client.py └── test_worker.py ├── themes └── theme.json └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | *.eggs 4 | .idea/ 5 | .tox/ 6 | build/ 7 | dist/ 8 | ambari_presto.egg-info/ 9 | docs/_build 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | install: 5 | - pip install -r requirements.txt 6 | script: 7 | - make dist test docs 8 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | Release History 4 | =============== 5 | 6 | 0.1.0 (TBD) 7 | --------------------- 8 | Initial release! -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include HISTORY.rst 2 | include LICENSE 3 | include README.md 4 | include metainfo.xml 5 | include Makefile 6 | include requirements.txt 7 | include package/scripts/download.ini 8 | 9 | recursive-include themes * 10 | recursive-include configuration * 11 | 12 | recursive-exclude * __pycache__ 13 | recursive-exclude * *.py[co] 14 | recursive-exclude tests * 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | .PHONY: clean-pyc clean-build clean-test test clean help 16 | 17 | help: 18 | @echo "clean-pyc - remove Python file artifacts" 19 | @echo "clean-build - remove build artifacts" 20 | @echo "clean-test - remove test artifacts" 21 | @echo "test - run all unit tests located in the tests directory" 22 | @echo "clean-docs - remove documentation artifacts" 23 | @echo "clean - remove all files and folders that are not checked into the repo" 24 | @echo "dist - package and build integration code in a tar.gz" 25 | @echo "docs - generate Sphinx HTML documentation" 26 | @echo "open-docs - open the root document (index.html) using xdg-open" 27 | @echo "help - display this help menu" 28 | 29 | clean: clean-pyc clean-build clean-test clean-docs 30 | 31 | clean-pyc: 32 | find . -name '*.pyc' -exec rm -f {} + 33 | find . -name '*.pyo' -exec rm -f {} + 34 | find . -name '*~' -exec rm -f {} + 35 | find . -name '__pycache__' -exec rm -fr {} + 36 | 37 | clean-build: 38 | rm -fr build/ 39 | rm -fr dist/ 40 | rm -fr ambari_presto.egg-info/ 41 | rm -fr .eggs/ 42 | 43 | clean-test: 44 | rm -rf .tox/ 45 | 46 | clean-docs: 47 | rm -rf docs/_build 48 | 49 | test: clean-test 50 | tox -- -s tests 51 | 52 | dist: clean-build clean-pyc 53 | ifdef VERSION 54 | sed -i 's%.*%$(VERSION)%' metainfo.xml 55 | endif 56 | python setup.py sdist 57 | ls -l dist 58 | 59 | docs: clean-docs 60 | $(MAKE) -C docs clean 61 | $(MAKE) -C docs html 62 | 63 | open-docs: 64 | xdg-open docs/_build/html/index.html 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ambari-presto-service [![BuildStatus](https://travis-ci.org/prestodb/ambari-presto-service.svg?branch=master)](https://travis-ci.org/prestodb/ambari-presto-service) 2 | 3 | This project contains the code and configuration needed to integrate Presto with Ambari. The 4 | full documentation can be found [here](https://prestodb.io/ambari-presto-service/). 5 | -------------------------------------------------------------------------------- /configuration/config.properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | node-scheduler.include-coordinator 21 | false 22 | 23 | Allow scheduling work on the coordinator. For larger clusters, processing 24 | work on the coordinator can impact query performance because the machine’s 25 | resources are not available for the critical task of scheduling, managing 26 | and monitoring query execution. 27 | 28 | 29 | value-list 30 | 31 | 32 | true 33 | 34 | 35 | 36 | false 37 | 38 | 39 | 40 | 1 41 | 42 | 43 | 44 | 45 | http-server.http.port 46 | 8285 47 | 48 | Specifies the port for the HTTP server. Presto uses HTTP for all 49 | communication, internal and external 50 | 51 | 52 | 53 | 54 | query.max-memory 55 | 50GB 56 | 57 | The maximum amount of distributed memory that a query may use. 58 | 59 | If you'd like to enter a value higher than the maximum on the slider, 60 | click on the pencil that appears when you hover over the setting and 61 | ignore that higher values are not recommended. 62 | 63 | 64 | int 65 | 0 66 | 300 67 | 2 68 | GB 69 | 70 | 71 | 72 | 73 | query.max-memory-per-node 74 | 1GB 75 | 76 | The maximum amount of memory that a query may use on any one machine. 77 | 78 | If you'd like to enter a value higher than the maximum on the slider, 79 | click on the pencil that appears when you hover over the setting and 80 | ignore that higher values are not recommended. 81 | 82 | 83 | int 84 | 0 85 | 100 86 | 1 87 | GB 88 | 89 | 90 | 91 | 92 | discovery.uri 93 | 94 | 95 | The URI to the Discovery server. Because we have enabled the embedded 96 | version of Discovery in the Presto coordinator, this should be the URI of 97 | the Presto coordinator. For example: http://master.net:8285 but make sure 98 | to replace master.net with the actual hostname of your coordinator node. 99 | Ensure the port in the URI matches the port set in the http-server.http.port 100 | property. This URI must not end in a slash. 101 | 102 | 103 | 104 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /configuration/connectors.properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | connectors.to.add 21 | {} 22 | 23 | Modify this property to add connectors. The format should be {'connector1': 24 | ['key1=value1', 'key2=value2'..], 'connector2' :['key1=value1', 25 | 'key2=value2'..]..}. Note the single quotes around each value! This example will 26 | create files connector1.properties, connector2.properties for Presto with entries 27 | key1=value1 etc. If you don't want to add any connectors then leave the value as {}; 28 | Ambari will not accept an empty value for this property. For deleting connectors, please 29 | use the connectors.to.remove property. 30 | 31 | 32 | 33 | connectors.to.delete 34 | [] 35 | 36 | Modify this property to delete connectors. The format should be 37 | ['connector1', 'connector2', 'connector3', ...]. Note the single quotes around each value! 38 | This example will delete the files connector1.properties, connector2.properties 39 | and connector3.properties in the Presto connector directory. If you don't want to delete 40 | any connectors then leave the value as []; Ambari will not accept an empty value for 41 | this property. 42 | 43 | 44 | -------------------------------------------------------------------------------- /configuration/jvm.config.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | jvm.config 21 | -server 22 | -Xmx16G 23 | -XX:+UseG1GC 24 | -XX:+UseGCOverheadLimit 25 | -XX:+ExplicitGCInvokesConcurrent 26 | -XX:+HeapDumpOnOutOfMemoryError 27 | -XX:OnOutOfMemoryError=kill -9 %p 28 | 29 | A list of command line options used for launching the Java Virtual 30 | Machine. The format of the file must be one option per line. These 31 | options are not interpreted by the shell, so options containing spaces or 32 | other special characters should not be quoted (as demonstrated by the 33 | OnOutOfMemoryError option). 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /configuration/node.properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | node.environment 21 | production 22 | 23 | The name of the environment. All Presto nodes in a cluster must have the 24 | same environment name. 25 | 26 | 27 | 28 | 42 | 43 | 44 | plugin.config-dir 45 | /etc/presto/catalog 46 | 47 | Configuration directory for plugins. This is where you should place 48 | connector property files. 49 | 50 | 51 | 52 | 53 | plugin.dir 54 | /usr/lib/presto/lib/plugin 55 | 56 | Library directory for plugins. 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Makefile for Sphinx documentation 16 | # 17 | 18 | # You can set these variables from the command line. 19 | SPHINXOPTS = 20 | SPHINXBUILD = sphinx-build 21 | PAPER = 22 | BUILDDIR = _build 23 | 24 | # User-friendly check for sphinx-build 25 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 26 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 27 | endif 28 | 29 | # Internal variables. 30 | PAPEROPT_a4 = -D latex_paper_size=a4 31 | PAPEROPT_letter = -D latex_paper_size=letter 32 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 33 | # the i18n builder cannot share the environment and doctrees with the others 34 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 35 | 36 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 37 | 38 | help: 39 | @echo "Please use \`make ' where is one of" 40 | @echo " html to make standalone HTML files" 41 | @echo " dirhtml to make HTML files named index.html in directories" 42 | @echo " singlehtml to make a single large HTML file" 43 | @echo " pickle to make pickle files" 44 | @echo " json to make JSON files" 45 | @echo " htmlhelp to make HTML files and a HTML help project" 46 | @echo " qthelp to make HTML files and a qthelp project" 47 | @echo " devhelp to make HTML files and a Devhelp project" 48 | @echo " epub to make an epub" 49 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 50 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 51 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 52 | @echo " text to make text files" 53 | @echo " man to make manual pages" 54 | @echo " texinfo to make Texinfo files" 55 | @echo " info to make Texinfo files and run them through makeinfo" 56 | @echo " gettext to make PO message catalogs" 57 | @echo " changes to make an overview of all changed/added/deprecated items" 58 | @echo " xml to make Docutils-native XML files" 59 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 60 | @echo " linkcheck to check all external links for integrity" 61 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 62 | 63 | clean: 64 | rm -rf $(BUILDDIR)/* 65 | 66 | html: 67 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 68 | @echo 69 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 70 | 71 | dirhtml: 72 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 73 | @echo 74 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 75 | 76 | singlehtml: 77 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 78 | @echo 79 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 80 | 81 | pickle: 82 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 83 | @echo 84 | @echo "Build finished; now you can process the pickle files." 85 | 86 | json: 87 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 88 | @echo 89 | @echo "Build finished; now you can process the JSON files." 90 | 91 | htmlhelp: 92 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 93 | @echo 94 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 95 | ".hhp project file in $(BUILDDIR)/htmlhelp." 96 | 97 | qthelp: 98 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 99 | @echo 100 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 101 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 102 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ambari-presto-service.qhcp" 103 | @echo "To view the help file:" 104 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ambari-presto-service.qhc" 105 | 106 | devhelp: 107 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 108 | @echo 109 | @echo "Build finished." 110 | @echo "To view the help file:" 111 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ambari-presto-service" 112 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ambari-presto-service" 113 | @echo "# devhelp" 114 | 115 | epub: 116 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 117 | @echo 118 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 119 | 120 | latex: 121 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 122 | @echo 123 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 124 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 125 | "(use \`make latexpdf' here to do that automatically)." 126 | 127 | latexpdf: 128 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 129 | @echo "Running LaTeX files through pdflatex..." 130 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 131 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 132 | 133 | latexpdfja: 134 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 135 | @echo "Running LaTeX files through platex and dvipdfmx..." 136 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 137 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 138 | 139 | text: 140 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 141 | @echo 142 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 143 | 144 | man: 145 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 146 | @echo 147 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 148 | 149 | texinfo: 150 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 151 | @echo 152 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 153 | @echo "Run \`make' in that directory to run these through makeinfo" \ 154 | "(use \`make info' here to do that automatically)." 155 | 156 | info: 157 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 158 | @echo "Running Texinfo files through makeinfo..." 159 | make -C $(BUILDDIR)/texinfo info 160 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 161 | 162 | gettext: 163 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 164 | @echo 165 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 166 | 167 | changes: 168 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 169 | @echo 170 | @echo "The overview file is in $(BUILDDIR)/changes." 171 | 172 | linkcheck: 173 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 174 | @echo 175 | @echo "Link check complete; look for any errors in the above output " \ 176 | "or in $(BUILDDIR)/linkcheck/output.txt." 177 | 178 | doctest: 179 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 180 | @echo "Testing of doctests in the sources finished, look at the " \ 181 | "results in $(BUILDDIR)/doctest/output.txt." 182 | 183 | xml: 184 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 185 | @echo 186 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 187 | 188 | pseudoxml: 189 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 190 | @echo 191 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 192 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # ambari-presto-service documentation build configuration file, created by 17 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 18 | # 19 | # This file is execfile()d with the current directory set to its 20 | # containing dir. 21 | # 22 | # Note that not all possible configuration values are present in this 23 | # autogenerated file. 24 | # 25 | # All configuration values have a default; values that are commented out 26 | # serve to show the default. 27 | 28 | import sys 29 | import os 30 | 31 | # If extensions (or modules to document with autodoc) are in another 32 | # directory, add these directories to sys.path here. If the directory is 33 | # relative to the documentation root, use os.path.abspath to make it 34 | # absolute, like shown here. 35 | #sys.path.insert(0, os.path.abspath('.')) 36 | 37 | # Get the project root dir, which is the parent dir of this 38 | cwd = os.getcwd() 39 | project_root = os.path.dirname(cwd) 40 | 41 | # Insert the project root dir as the first element in the PYTHONPATH. 42 | # This lets us ensure that the source package is imported, and that its 43 | # version is used. 44 | sys.path.insert(0, project_root) 45 | 46 | # import ambari-presto-service 47 | 48 | # -- General configuration --------------------------------------------- 49 | 50 | # If your documentation needs a minimal Sphinx version, state it here. 51 | #needs_sphinx = '1.0' 52 | 53 | # Add any Sphinx extension module names here, as strings. They can be 54 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 55 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 56 | 57 | # Add any paths that contain templates here, relative to this directory. 58 | templates_path = ['_templates'] 59 | 60 | # The suffix of source filenames. 61 | source_suffix = '.rst' 62 | 63 | # The encoding of source files. 64 | #source_encoding = 'utf-8-sig' 65 | 66 | # The master toctree document. 67 | master_doc = 'index' 68 | 69 | # General information about the project. 70 | project = u'ambari-presto-service' 71 | 72 | # The version info for the project you're documenting, acts as replacement 73 | # for |version| and |release|, also used in various other places throughout 74 | # the built documents. 75 | # 76 | # The short X.Y version. 77 | version = '1.3' 78 | # The full version, including alpha/beta/rc tags. 79 | release = '1.3' 80 | 81 | # The language for content autogenerated by Sphinx. Refer to documentation 82 | # for a list of supported languages. 83 | #language = None 84 | 85 | # There are two options for replacing |today|: either, you set today to 86 | # some non-false value, then it is used: 87 | #today = '' 88 | # Else, today_fmt is used as the format for a strftime call. 89 | #today_fmt = '%B %d, %Y' 90 | 91 | # List of patterns, relative to source directory, that match files and 92 | # directories to ignore when looking for source files. 93 | exclude_patterns = ['_build'] 94 | 95 | # The reST default role (used for this markup: `text`) to use for all 96 | # documents. 97 | #default_role = None 98 | 99 | # If true, '()' will be appended to :func: etc. cross-reference text. 100 | #add_function_parentheses = True 101 | 102 | # If true, the current module name will be prepended to all description 103 | # unit titles (such as .. function::). 104 | #add_module_names = True 105 | 106 | # If true, sectionauthor and moduleauthor directives will be shown in the 107 | # output. They are ignored by default. 108 | #show_authors = False 109 | 110 | # The name of the Pygments (syntax highlighting) style to use. 111 | pygments_style = 'sphinx' 112 | 113 | # A list of ignored prefixes for module index sorting. 114 | #modindex_common_prefix = [] 115 | 116 | # If true, keep warnings as "system message" paragraphs in the built 117 | # documents. 118 | #keep_warnings = False 119 | 120 | 121 | # -- Options for HTML output ------------------------------------------- 122 | 123 | # The theme to use for HTML and HTML Help pages. See the documentation for 124 | # a list of builtin themes. 125 | html_theme = 'default' 126 | 127 | # Theme options are theme-specific and customize the look and feel of a 128 | # theme further. For a list of options available for each theme, see the 129 | # documentation. 130 | #html_theme_options = {} 131 | 132 | # Add any paths that contain custom themes here, relative to this directory. 133 | #html_theme_path = [] 134 | 135 | # The name for this set of Sphinx documents. If None, it defaults to 136 | # " v documentation". 137 | #html_title = None 138 | 139 | # A shorter title for the navigation bar. Default is the same as 140 | # html_title. 141 | #html_short_title = None 142 | 143 | # The name of an image file (relative to this directory) to place at the 144 | # top of the sidebar. 145 | #html_logo = None 146 | 147 | # The name of an image file (within the static path) to use as favicon 148 | # of the docs. This file should be a Windows icon file (.ico) being 149 | # 16x16 or 32x32 pixels large. 150 | #html_favicon = None 151 | 152 | # Add any paths that contain custom static files (such as style sheets) 153 | # here, relative to this directory. They are copied after the builtin 154 | # static files, so a file named "default.css" will overwrite the builtin 155 | # "default.css". 156 | html_static_path = ['_static'] 157 | 158 | # If not '', a 'Last updated on:' timestamp is inserted at every page 159 | # bottom, using the given strftime format. 160 | #html_last_updated_fmt = '%b %d, %Y' 161 | 162 | # If true, SmartyPants will be used to convert quotes and dashes to 163 | # typographically correct entities. 164 | #html_use_smartypants = True 165 | 166 | # Custom sidebar templates, maps document names to template names. 167 | #html_sidebars = {} 168 | 169 | # Additional templates that should be rendered to pages, maps page names 170 | # to template names. 171 | #html_additional_pages = {} 172 | 173 | # If false, no module index is generated. 174 | #html_domain_indices = True 175 | 176 | # If false, no index is generated. 177 | #html_use_index = True 178 | 179 | # If true, the index is split into individual pages for each letter. 180 | #html_split_index = False 181 | 182 | # If true, links to the reST sources are added to the pages. 183 | #html_show_sourcelink = True 184 | 185 | # If true, "Created using Sphinx" is shown in the HTML footer. 186 | # Default is True. 187 | #html_show_sphinx = True 188 | 189 | # If true, "(C) Copyright ..." is shown in the HTML footer. 190 | # Default is True. 191 | html_show_copyright = False 192 | 193 | # If true, an OpenSearch description file will be output, and all pages 194 | # will contain a tag referring to it. The value of this option 195 | # must be the base URL from which the finished HTML is served. 196 | #html_use_opensearch = '' 197 | 198 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 199 | #html_file_suffix = None 200 | 201 | # Output file base name for HTML help builder. 202 | htmlhelp_basename = 'ambari-presto-servicedoc' 203 | 204 | 205 | # -- Options for LaTeX output ------------------------------------------ 206 | 207 | latex_elements = { 208 | # The paper size ('letterpaper' or 'a4paper'). 209 | #'papersize': 'letterpaper', 210 | 211 | # The font size ('10pt', '11pt' or '12pt'). 212 | #'pointsize': '10pt', 213 | 214 | # Additional stuff for the LaTeX preamble. 215 | #'preamble': '', 216 | } 217 | 218 | # Grouping the document tree into LaTeX files. List of tuples 219 | # (source start file, target name, title, author, documentclass 220 | # [howto/manual]). 221 | latex_documents = [ 222 | ('index', 'ambari-presto-service.tex', 223 | u'ambari-presto-service Documentation', 224 | 'manual'), 225 | ] 226 | 227 | # The name of an image file (relative to this directory) to place at 228 | # the top of the title page. 229 | #latex_logo = None 230 | 231 | # For "manual" documents, if this is true, then toplevel headings 232 | # are parts, not chapters. 233 | #latex_use_parts = False 234 | 235 | # If true, show page references after internal links. 236 | #latex_show_pagerefs = False 237 | 238 | # If true, show URL addresses after external links. 239 | #latex_show_urls = False 240 | 241 | # Documents to append as an appendix to all manuals. 242 | #latex_appendices = [] 243 | 244 | # If false, no module index is generated. 245 | #latex_domain_indices = True 246 | 247 | 248 | # -- Options for manual page output ------------------------------------ 249 | 250 | # One entry per manual page. List of tuples 251 | # (source start file, name, description, authors, manual section). 252 | man_pages = [ 253 | ('index', 'ambari-presto-service', 254 | u'ambari-presto-service Documentation', 255 | 1) 256 | ] 257 | 258 | # If true, show URL addresses after external links. 259 | #man_show_urls = False 260 | 261 | 262 | # -- Options for Texinfo output ---------------------------------------- 263 | 264 | # Grouping the document tree into Texinfo files. List of tuples 265 | # (source start file, target name, title, author, 266 | # dir menu entry, description, category) 267 | texinfo_documents = [ 268 | ('index', 'ambari-presto-service', 269 | u'ambari-presto-service Documentation', 270 | 'ambari-presto-service', 271 | 'One line description of project.', 272 | 'Miscellaneous'), 273 | ] 274 | 275 | # Documents to append as an appendix to all manuals. 276 | #texinfo_appendices = [] 277 | 278 | # If false, no module index is generated. 279 | #texinfo_domain_indices = True 280 | 281 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 282 | #texinfo_show_urls = 'footnote' 283 | 284 | # If true, do not generate a @detailmenu in the "Top" node's menu. 285 | #texinfo_no_detailmenu = False 286 | -------------------------------------------------------------------------------- /docs/developers.rst: -------------------------------------------------------------------------------- 1 | Developers 2 | ========== 3 | 4 | Requirements for development 5 | ---------------------------- 6 | 7 | 1. Python 2.6/2.7. 8 | 2. `pip `_. 9 | 3. `make `_. 10 | 4. ``pip install -r $(REPO_ROOT_DIR)/requirements.txt``. 11 | 12 | Definitions 13 | ----------- 14 | 15 | The following definitions, taken from the `Apache Ambari wiki `_, 16 | are useful when talking about integration: 17 | 18 | 1. Stack - Defines a set of Services and the location where to obtain the 19 | software packages for those Services. A Stack can have one or more version, 20 | and each version can be active/inactive. For example, Stack = HDP-2.3. 21 | 2. Service - Defines the Components (MASTER, SLAVE, CLIENT) that make up the 22 | Service. For example, Service = HDFS. 23 | 3. Component - The individual Components that adhere to a certain defined 24 | lifecycle (start, stop, install, etc). For example, Service = HDFS has 25 | Components = NameNode (MASTER), Secondary NameNode (MASTER), DataNode 26 | (SLAVE) and HDFS Client (CLIENT). 27 | 28 | Information on integrating services with Ambari 29 | ----------------------------------------------- 30 | 31 | For more information on developing service integration code for Ambari, the 32 | following resources might be helpful: 33 | 34 | 1. `Webcast `_ with Hortonworks 35 | engineers about integrating with Ambari. Includes slides and recorded 36 | video/audio of the talk. 37 | 2. Lots of `integration examples `_. 38 | 39 | .. _build_and_custom_distributions: 40 | 41 | Build and custom distributions 42 | ------------------------------ 43 | 44 | The build system for this project is very simple; it uses Python's standard 45 | `distutils `_ module. Calls to the 46 | ``setup.py`` script are wrapped with a Makefile to make common operations 47 | simple. Execute ``make dist`` to build the distribution, ``make test`` to run 48 | the unit tests and ``make help`` to get more info on all the available 49 | targets. 50 | 51 | By default, the integration code installs Presto version ``0.196``. Change the 52 | version displayed by Ambari when adding the Presto service by specifying a 53 | value for the ``VERSION`` variable when building the distribution. For 54 | example, to display Presto version ``0.134``, run ``make dist VERSION=0.134``. 55 | To download a different RPM and CLI to match version ``0.134``, edit the 56 | ``package/scripts/download.ini`` file with URLs for both. 57 | -------------------------------------------------------------------------------- /docs/getting-help.rst: -------------------------------------------------------------------------------- 1 | Getting Help 2 | ============ 3 | 4 | If you're having trouble with anything, please 5 | `file a new issue `_ 6 | and we'll try our best to help you out. -------------------------------------------------------------------------------- /docs/getting-started.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started: 2 | 3 | =============== 4 | Getting Started 5 | =============== 6 | 7 | Requirements for integration 8 | ============================ 9 | 10 | 1. Red Hat Enterprise Linux 6.x (64-bit) or CentOS equivalent. 11 | 2. You must have Ambari installed and thus transitively fulfill 12 | `Ambari's requirements `_. 13 | 3. Oracle's JDK 1.8u60+ for Presto version 0.148 and over, and 14 | 1.8u40+ for Presto versions before 0.148. We recommend you read the section on 15 | :ref:`jdk-configuration-label` to fully understand 16 | the relationship between Presto and Ambari's JDK. 17 | 4. Disable ``requiretty``. On RHEL 6.x this can be done by editing the 18 | ``/etc/sudoers`` file and commenting out ``Defaults requiretty``. 19 | 5. Install ``wget`` on all nodes that will run a Presto component. 20 | 21 | Adding the Presto service 22 | ========================= 23 | 24 | This section and all others that follow within :ref:`Getting Started ` 25 | walk you through the integration steps needed to get Presto working with 26 | Ambari. By default, this integration code installs Presto version ``0.196``, 27 | the latest version at the time of writing. To install the latest Teradata 28 | Presto release (``0.148t`` at the time of writing), download the Ambari 29 | integration package from `here `_ and follow 30 | the remaining instructions below. To change the distribution to install 31 | another version, see :ref:`Build and custom distributions `. 32 | 33 | To integrate the Presto service with Ambari, follow the steps outlined below: 34 | 35 | * Assuming HDP 2.6 was installed with Ambari, create the following directory on 36 | the node where the ``ambari-server`` is running: 37 | 38 | .. code-block:: bash 39 | 40 | $ mkdir /var/lib/ambari-server/resources/stacks/HDP/2.6/services/PRESTO 41 | $ cd /var/lib/ambari-server/resources/stacks/HDP/2.6/services/PRESTO 42 | 43 | * Place the integration files within the newly created PRESTO directory. 44 | Download the integration package that installs Teradata's version from 45 | `here `_ or download the integration package 46 | that installs ``0.148`` from the `releases section of this project `_. Upload the integration archive to your cluster and extract it like so: 47 | 48 | .. code-block:: bash 49 | 50 | $ tar -xvf /path/to/integration/package/ambari-presto-1.3.tar.gz -C /var/lib/ambari-server/resources/stacks/HDP/2.6/services/PRESTO 51 | $ mv /var/lib/ambari-server/resources/stacks/HDP/2.6/services/PRESTO/ambari-presto-1.3/* /var/lib/ambari-server/resources/stacks/HDP/2.6/services/PRESTO 52 | $ rm -rf /var/lib/ambari-server/resources/stacks/HDP/2.6/services/PRESTO/ambari-presto-1.3 53 | 54 | * Finally, make all integration files executable and restart the Ambari server: 55 | 56 | .. code-block:: bash 57 | 58 | $ chmod -R +x /var/lib/ambari-server/resources/stacks/HDP/2.6/services/PRESTO/* 59 | $ ambari-server restart 60 | 61 | * Once the server has restarted, point your browser to it and on the main 62 | Ambari Web UI page click the ``Add Service`` button and follow the on 63 | screen wizard to add Presto. The following sections provide more details 64 | on the options and choices you will make when adding Presto. 65 | 66 | Supported topologies 67 | ==================== 68 | 69 | The following two screens will allow you to assign the Presto processes among 70 | the nodes in your cluster. Once you pick a topology for Presto and finish the 71 | installation process it is impossible to modify that topology. 72 | 73 | Presto is composed of a coordinator and worker processes. The same code runs 74 | all nodes because the same Presto server RPM is installed for both workers and 75 | coordinator. It is the configuration on each node that determines how a 76 | particular node will behave. Presto can run in pseudo-distributed mode, where 77 | a single Presto process on one node acts as both coordinator and worker, or it 78 | can run in distributed mode, where the Presto coordinator runs on one node and 79 | the Presto workers run on other nodes. 80 | 81 | The client component of Presto is the ``presto-cli`` executable JAR. You 82 | should place it on all nodes where you expect to access the Presto server via 83 | this command line utility. The ``presto-cli`` executable JAR does not need to 84 | be co-located with either a worker or a coordinator, it can be installed on 85 | its own. Once installed, the CLI can be found at 86 | ``/usr/lib/presto/bin/presto-cli``. 87 | 88 | **Do not place a worker on the same node as a coordinator.** Such an attempt 89 | will fail the installation because the integration software will attempt to 90 | install the RPM twice. In order to schedule work on the Presto coordinator, 91 | effectively turning the process into a dual worker/coordinator, please enable 92 | the ``node-scheduler.include-coordinator`` toggle available in the 93 | configuration screen. 94 | 95 | Pseudo-distributed 96 | ------------------ 97 | 98 | Pick a node for the Presto coordinator and **do not assign any Presto workers**. 99 | On the configuration screen that follows, you must also enable 100 | ``node-scheduler.include-coordinator`` by clicking the toggle. 101 | 102 | Distributed 103 | ----------- 104 | 105 | Pick a node for the Presto coordinator and assign as many Presto workers to 106 | nodes as you'd like. Feel free to also place the client component on any node. 107 | Remember to not place a worker on the same node as a coordinator. 108 | 109 | Configuring Presto 110 | ================== 111 | 112 | The one configuration property that does not have a default and requires 113 | input is ``discovery.uri``. The expected value is 114 | ``http://:8285``. Note that it is **http** 115 | and not **https** and that the port is 8285. If you change the value of 116 | ``http-server.http.port``, make sure to also change it in ``disovery.uri``. 117 | 118 | Some of the most popular properties are displayed in the Settings tab 119 | (open by default). In the Advanced tab, set custom properties by opening up 120 | the correct drop down and specifying a key and a value. Note that specifying 121 | a property that Presto does not recognize will cause the installation to 122 | finish with errors as some or all servers fail to start. 123 | 124 | Change the Presto configuration after installation by selecting the Presto 125 | service followed by the Configs tab. After changing a configuration option, 126 | make sure to restart Presto for the changes to take effect. 127 | 128 | If you are running a version of Ambari that is older than 2.1 129 | (version number numerically less than 2.1), then you must omit the memory 130 | suffix (GB) when setting the following memory related configurations: 131 | ``query.max-memory-per-node`` and ``query.max-memory``. For these two 132 | properties the memory suffix is automatically added by the integration 133 | software. For all other memory related configurations that you add as 134 | custom properties, you'll have to include the memory suffix when specifying 135 | the value. 136 | 137 | Adding and removing connectors 138 | ------------------------------ 139 | 140 | To add a connector modify the ``connectors.to.add`` property, whose format is 141 | the following: ``{'connector1': ['key1=value1', 'key2=value2', etc.], 142 | 'connector2': ['key3=value3', 'key4=value4'], etc.}``. 143 | Note the single quotes around each individual element. This property only 144 | adds connectors and will not delete connectors. Thus, if you add 145 | ``connector1``, save the configuration, restart Presto, then specify ``{}`` 146 | for this property, ``connector1`` will not be deleted. If you specify 147 | incorrect values in your connector settings, for example setting the 148 | ``hive.metastore.uri`` in the Hive connector to point to an invalid hostname, 149 | then Presto will fail to start. 150 | 151 | For example, to add the Hive and Kafka connectors, set the `connectors.to.add` property to: 152 | 153 | .. code-block:: none 154 | 155 | { 156 | 'hive': ['connector.name=hive-cdh4', 'hive.metastore.uri=thrift://example.net:9083'], 157 | 'kafka': ['connector.name=kafka', 'kafka.table-names=table1,table2', 'kafka.nodes=host1:port,host2:port'] 158 | } 159 | 160 | To delete a connector modify the ``connectors.to.delete`` property, whose 161 | format is the following: ``['connector1', 'connector2', etc.]``. Again, 162 | note the single quotes around each element. The above value will delete 163 | connectors ``connector1`` and ``connector2``. Note that the ``tpch`` 164 | connector cannot be deleted because it is used to smoketest Presto after 165 | it starts. The presence of the ``tpch`` connector has negligible impact on 166 | the system. 167 | 168 | For example, to delete the Hive and Kafka connectors, set the 169 | ``connectors.to.delete`` property to: ``['hive', 'kafka']``. 170 | 171 | .. _jdk-configuration-label: 172 | 173 | JDK Configuration 174 | ================= 175 | 176 | During Ambari's installation, the user is allowed to pick the JDK 177 | that Ambari will use to start itself as well as other services it controls. 178 | This JDK can be edited at any time after installation by running 179 | ``ambari-server setup`` on the host running the Ambari server process and 180 | then restarting that process by running ``ambari-server restart`` for 181 | the changes to take effect. 182 | 183 | When choosing the JDK version to run, the user is presented with three 184 | options: ``1.8``, ``1.7`` or a custom JDK. If the ``1.8`` or ``1.7`` 185 | option is chosen then Ambari will download a JDK of that major version. 186 | However, the update (minor) versions of the JDK differs based on Ambari's 187 | version. For example, Ambari ``2.2.0+`` will download ``1.8u60`` and 188 | versions before will download ``1.8u40``. 189 | 190 | When Ambari installs Presto, the JDK used is going to be the JDK 191 | that Ambari was configured with (specifically, the value of 192 | ``java.home`` in ``/etc/ambari-server/conf/ambari.properties``). 193 | However, unlike other services, once Presto is installed it will 194 | use the same JDK it was installed with even if Ambari's JDK 195 | is re-configured. The reason for this is that during RPM installation, 196 | Presto's JDK is set in ``/etc/presto/env.sh``. To 197 | re-configure Presto's JDK, edit ``/etc/presto/env.sh`` on all 198 | hosts where Presto will run. 199 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ambari-presto-service 2 | ===================== 3 | 4 | `Issues `_ | 5 | `Github `_ 6 | 7 | Introduction 8 | ------------ 9 | 10 | This project contains the code and configuration needed to integrate 11 | `Presto `_ with `Ambari `_. 12 | Adding the Presto service to Ambari allows: 13 | 14 | 1. Installing and deploying Presto on a cluster from the Ambari UI. 15 | 2. Changing Presto configuration options via the Ambari UI. 16 | 17 | Content 18 | ------- 19 | 20 | .. toctree:: 21 | :maxdepth: 3 22 | 23 | getting-started 24 | known-issues 25 | getting-help 26 | developers -------------------------------------------------------------------------------- /docs/known-issues.rst: -------------------------------------------------------------------------------- 1 | Known Issues 2 | ============ 3 | 4 | * For some older versions of Presto, when attempting to ``CREATE TABLE`` or 5 | ``CREATE TABLE AS`` using the Hive connector, you may run into the following 6 | error:: 7 | 8 | Query 20151120_203243_00003_68gdx failed: java.security.AccessControlException: Permission denied: user=hive, access=WRITE, inode="/apps/hive/warehouse/nation":hdfs:hdfs:drwxr-xr-x 9 | at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:319) 10 | at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:219) 11 | at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:190) 12 | at org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1771) 13 | 14 | To work around the issue, edit your ``jvm.config`` settings by adding the 15 | following property ``-DHADOOP_USER_NAME=hive``. This problem affects Presto 16 | ``0.115t`` but does not affect ``0.127t``. After saving your edit to 17 | ``jvm.config``, don't forget to restart all Presto components in order for 18 | the changes to take effect. 19 | 20 | * If you decide to deploy an older version of Presto, you may have to adjust 21 | some setting manually. Please see [Configuring Presto](#configuring-presto) 22 | for an explanation of how to add custom settings. For example, the 23 | ``task.max-memory`` setting was deprecated in ``0.127t`` but is valid in 24 | ``0.115t``. Therefore, if you're installing ``0.115t`` and would like to 25 | change ``task.max-memory`` to something other than its default, add it as 26 | a custom property. 27 | 28 | * On the Presto service home page, if you click on ``Presto workers``, you 29 | will get an incorrect list of workers. This is a known issue and has been 30 | fixed in Ambari 2.2.0. 31 | 32 | * If the installation of Presto fails with ``Python script has been killed 33 | due to timeout after waiting 1800 secs``, then the ``wget`` for either the 34 | Presto RPM or ``presto-cli`` JAR has timed out. To increase the timeout, 35 | increase the ``agent.package.install.task.timeout`` setting in 36 | ``/etc/ambari-server/conf/ambari.properties`` on the Ambari server host. 37 | Make sure to restart the Ambari server for the change to take effect. 38 | To resume, either hit the Retry button in the installation wizard, or 39 | finish the wizard and then install all Presto components individually by 40 | navigating to the relevant host and selecting ``re-install``. The 41 | components can be installed manually in any order but when starting the 42 | components, make sure to start the Presto coordinator last. If the 43 | installation keeps timing out, we suggest downloading the RPM and JAR 44 | outside the installation process, uploading them somewhere on your network 45 | and editing ``package/scripts/download.ini`` with the new URLs. 46 | 47 | * At the moment, upgrading Presto from Ambari is not possible. Even though 48 | Ambari provides the capability to upgrade software, we didn't get a chance 49 | to implement the integration. If many users request this feature 50 | (if you'd like to see this feature let us know by commenting on 51 | `this issue `_), 52 | we'll add it to the next release. To upgrade Presto without the native 53 | upgrade integration you have to manually uninstall Presto and then install 54 | the new version. -------------------------------------------------------------------------------- /metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 2.0 19 | 20 | 21 | PRESTO 22 | Presto 23 | Presto is an open source distributed SQL query engine for running 24 | interactive analytic queries against data sources of all sizes ranging 25 | from gigabytes to petabytes. 26 | 27 | 0.196 28 | 29 | 30 | PRESTO_COORDINATOR 31 | Presto coordinator 32 | MASTER 33 | 1 34 | true 35 | 36 | 37 | PYTHON 38 | 1200 39 | 40 | 41 | 42 | 43 | PRESTO_WORKER 44 | Presto worker 45 | SLAVE 46 | 0+ 47 | true 48 | 49 | 50 | PYTHON 51 | 52 | 53 | 54 | 55 | PRESTO_CLI 56 | Presto command line interface 57 | CLIENT 58 | 1+ 59 | true 60 | 61 | 62 | PYTHON 63 | 64 | 65 | 66 | 67 | 68 | node.properties 69 | config.properties 70 | jvm.config 71 | connectors.properties 72 | 73 | 74 | 75 | 76 | theme.json 77 | true 78 | 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /package/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/package/__init__.py -------------------------------------------------------------------------------- /package/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/package/scripts/__init__.py -------------------------------------------------------------------------------- /package/scripts/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import ast 17 | import ConfigParser 18 | 19 | from resource_management.core.resources.system import Execute 20 | 21 | script_dir = os.path.dirname(os.path.realpath(__file__)) 22 | config = ConfigParser.ConfigParser() 23 | config.readfp(open(os.path.join(script_dir, 'download.ini'))) 24 | 25 | PRESTO_RPM_URL = config.get('download', 'presto_rpm_url') 26 | PRESTO_RPM_NAME = PRESTO_RPM_URL.split('/')[-1] 27 | PRESTO_CLI_URL = config.get('download', 'presto_cli_url') 28 | 29 | def create_connectors(node_properties, connectors_to_add): 30 | if not connectors_to_add: 31 | return 32 | Execute('mkdir -p {0}'.format(node_properties['plugin.config-dir'])) 33 | connectors_dict = ast.literal_eval(connectors_to_add) 34 | for connector in connectors_dict: 35 | connector_file = os.path.join(node_properties['plugin.config-dir'], connector + '.properties') 36 | with open(connector_file, 'w') as f: 37 | for lineitem in connectors_dict[connector]: 38 | f.write('{0}\n'.format(lineitem)) 39 | 40 | def delete_connectors(node_properties, connectors_to_delete): 41 | if not connectors_to_delete: 42 | return 43 | connectors_list = ast.literal_eval(connectors_to_delete) 44 | for connector in connectors_list: 45 | connector_file_name = os.path.join(node_properties['plugin.config-dir'], connector + '.properties') 46 | Execute('rm -f {0}'.format(connector_file_name)) 47 | -------------------------------------------------------------------------------- /package/scripts/download.ini: -------------------------------------------------------------------------------- 1 | [download] 2 | presto_rpm_url = http://search.maven.org/remotecontent?filepath=com/facebook/presto/presto-server-rpm/0.196/presto-server-rpm-0.196.rpm 3 | presto_cli_url = http://search.maven.org/remotecontent?filepath=com/facebook/presto/presto-cli/0.196/presto-cli-0.196-executable.jar 4 | -------------------------------------------------------------------------------- /package/scripts/params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from resource_management.libraries.script.script import Script 17 | 18 | # config object that holds the configurations declared in the config xml file 19 | config = Script.get_config() 20 | 21 | node_properties = config['configurations']['node.properties'] 22 | jvm_config = config['configurations']['jvm.config'] 23 | config_properties = config['configurations']['config.properties'] 24 | 25 | connectors_to_add = config['configurations']['connectors.properties']['connectors.to.add'] 26 | connectors_to_delete = config['configurations']['connectors.properties']['connectors.to.delete'] 27 | 28 | daemon_control_script = '/etc/init.d/presto' 29 | config_directory = '/etc/presto' 30 | 31 | memory_configs = ['query.max-memory-per-node', 'query.max-memory'] 32 | 33 | host_info = config['clusterHostInfo'] 34 | 35 | host_level_params = config['hostLevelParams'] 36 | java_home = host_level_params['java_home'] 37 | -------------------------------------------------------------------------------- /package/scripts/presto_cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from resource_management.libraries.script.script import Script 16 | from resource_management.core.resources.system import Execute 17 | from resource_management.core.exceptions import ClientComponentHasNoStatus 18 | from common import PRESTO_CLI_URL 19 | 20 | 21 | class Cli(Script): 22 | def install(self, env): 23 | Execute('mkdir -p /usr/lib/presto/bin') 24 | Execute('wget --no-check-certificate {0} -O /usr/lib/presto/bin/presto-cli'.format(PRESTO_CLI_URL)) 25 | Execute('chmod +x /usr/lib/presto/bin/presto-cli') 26 | 27 | def status(self, env): 28 | raise ClientComponentHasNoStatus() 29 | 30 | def configure(self, env): 31 | import params 32 | env.set_params(params) 33 | 34 | def start(self, env): 35 | import params 36 | env.set_params(params) 37 | 38 | def stop(self, env): 39 | import params 40 | env.set_params(params) 41 | 42 | if __name__ == '__main__': 43 | Cli().execute() -------------------------------------------------------------------------------- /package/scripts/presto_client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Simple client to communicate with a Presto server. 17 | """ 18 | from httplib import HTTPConnection, HTTPException 19 | import logging 20 | import json 21 | import socket 22 | import sys 23 | import time 24 | 25 | from urllib2 import HTTPError, urlopen, URLError 26 | 27 | 28 | URL_TIMEOUT_MS = 5000 29 | NUM_ROWS = 1000 30 | DATA_RESP = 'data' 31 | NEXT_URI_RESP = 'nextUri' 32 | RETRY_TIMEOUT = 120 33 | SYSTEM_RUNTIME_NODES = 'select * from system.runtime.nodes' 34 | SHOW_CATALOGS = 'show catalogs' 35 | SLEEP_INTERVAL = 10 36 | 37 | logging.basicConfig(stream=sys.stdout) 38 | _LOGGER = logging.getLogger(__name__) 39 | 40 | def smoketest_presto(client, all_hosts): 41 | ensure_nodes_are_up(client, all_hosts) 42 | ensure_catalogs_are_available(client) 43 | client.execute_query('select * from nation', schema='sf1', catalog='tpch') 44 | rows = client.get_rows() 45 | if len(rows) != 25: 46 | raise RuntimeError('Presto server failed to return the correct \ 47 | number of rows from nation table in TPCH connector. Expected 25 but got {0}'.format(len(rows))) 48 | 49 | def ensure_catalogs_are_available(client): 50 | rows = [] 51 | elapsed_time = 0 52 | while elapsed_time < RETRY_TIMEOUT: 53 | client.execute_query(SHOW_CATALOGS) 54 | rows = client.get_rows() 55 | if not rows: 56 | time.sleep(SLEEP_INTERVAL) 57 | _LOGGER.debug('Failed to load catalogs after ' 58 | 'waiting for %d seconds. Retrying...' % elapsed_time) 59 | elapsed_time += SLEEP_INTERVAL 60 | else: 61 | break 62 | if not rows: 63 | raise RuntimeError('Presto server failed to load all catalogs within \ 64 | {0} seconds.'.format(RETRY_TIMEOUT)) 65 | 66 | 67 | def ensure_nodes_are_up(client, all_hosts): 68 | result = True 69 | elapsed_time = 0 70 | while elapsed_time < RETRY_TIMEOUT: 71 | result = client.execute_query(SYSTEM_RUNTIME_NODES) 72 | if not result: 73 | time.sleep(SLEEP_INTERVAL) 74 | _LOGGER.debug('Status retrieval for the server failed after ' 75 | 'waiting for %d seconds. Retrying...' % elapsed_time) 76 | elapsed_time += SLEEP_INTERVAL 77 | else: 78 | break 79 | if not result: 80 | raise RuntimeError('Presto server failed to start within \ 81 | {0} seconds.'.format(RETRY_TIMEOUT)) 82 | 83 | # Verify that the nodes we expect to have registered with the Discovery 84 | # service have actually registered correctly 85 | elapsed_time = 0 86 | are_expected_nodes_up = False 87 | while elapsed_time < RETRY_TIMEOUT: 88 | client.execute_query(SYSTEM_RUNTIME_NODES) 89 | nodes_returned_from_presto = [] 90 | for row in client.get_rows(): 91 | nodes_returned_from_presto.append(row[0]) 92 | if len(nodes_returned_from_presto) == len(all_hosts): 93 | are_expected_nodes_up = True 94 | break 95 | else: 96 | time.sleep(SLEEP_INTERVAL) 97 | _LOGGER.debug('Elapsed time {0}'.format(elapsed_time)) 98 | _LOGGER.debug( 99 | 'Number of hosts returned from Presto {0} do not match number of hosts specified by user {1}'.format( 100 | nodes_returned_from_presto, all_hosts)) 101 | elapsed_time += SLEEP_INTERVAL 102 | if not are_expected_nodes_up: 103 | raise RuntimeError( 104 | 'Number of hosts returned from Presto {0} do not equal the number of hosts specified by user {1}'.format( 105 | nodes_returned_from_presto, all_hosts)) 106 | 107 | # This class was copied more or less verbatim from 108 | # https://github.com/prestodb/presto-admin/blob/master/prestoadmin/prestoclient.py 109 | class PrestoClient: 110 | response_from_server = {} 111 | # rows returned by the query 112 | rows = [] 113 | next_uri = '' 114 | 115 | def __init__(self, server, user, port=None): 116 | self.server = server 117 | self.user = user 118 | self.port = port if port else None 119 | 120 | def clear_old_results(self): 121 | if self.rows: 122 | self.rows = [] 123 | 124 | if self.next_uri: 125 | self.next_uri = '' 126 | 127 | if self.response_from_server: 128 | self.response_from_server = {} 129 | 130 | def execute_query(self, sql, schema='sf1', catalog='tpch'): 131 | """ 132 | Execute a query connecting to Presto server using passed parameters. 133 | 134 | Client sends http POST request to the Presto server, page: 135 | '/v1/statement'. Header information should 136 | include: X-Presto-Catalog, X-Presto-Schema, X-Presto-User 137 | 138 | Args: 139 | sql: SQL query to be executed 140 | schema: Presto schema to be used while executing query 141 | (default=default) 142 | catalog: Catalog to be used by the server 143 | 144 | Returns: 145 | True or False exit status 146 | """ 147 | if not sql: 148 | raise InvalidArgumentError('SQL query missing') 149 | 150 | if not self.server: 151 | raise InvalidArgumentError('Server IP missing') 152 | 153 | if not self.user: 154 | raise InvalidArgumentError('Username missing') 155 | 156 | if not self.port: 157 | raise InvalidArgumentError('Port missing') 158 | 159 | self.clear_old_results() 160 | 161 | headers = {'X-Presto-Catalog': catalog, 162 | 'X-Presto-Schema': schema, 163 | 'X-Presto-User': self.user} 164 | answer = '' 165 | try: 166 | _LOGGER.info('Connecting to server at: ' + self.server + 167 | ':' + str(self.port) + ' as user ' + self.user) 168 | conn = HTTPConnection(self.server, int(self.port), False, 169 | URL_TIMEOUT_MS) 170 | conn.request('POST', '/v1/statement', sql, headers) 171 | response = conn.getresponse() 172 | 173 | if response.status != 200: 174 | conn.close() 175 | _LOGGER.error('Connection error: ' 176 | + str(response.status) + ' ' + response.reason) 177 | return False 178 | 179 | answer = response.read() 180 | conn.close() 181 | 182 | self.response_from_server = json.loads(answer) 183 | _LOGGER.info('Query executed successfully') 184 | return True 185 | except (HTTPException, socket.error): 186 | _LOGGER.error('Error connecting to presto server at: ' + 187 | self.server + ':' + str(self.port)) 188 | return False 189 | except ValueError as e: 190 | _LOGGER.error('Error connecting to Presto server: ' + str(e) + 191 | ' error from server: ' + answer) 192 | raise e 193 | 194 | def get_response_from(self, uri): 195 | """ 196 | Sends a GET request to the Presto server at the specified next_uri 197 | and updates the response 198 | """ 199 | try: 200 | conn = urlopen(uri, None, URL_TIMEOUT_MS) 201 | answer = conn.read() 202 | conn.close() 203 | 204 | self.response_from_server = json.loads(answer) 205 | _LOGGER.info('GET request successful for uri: ' + uri) 206 | return True 207 | except (HTTPError, URLError) as e: 208 | _LOGGER.error('Error opening the presto response uri: ' + 209 | str(e.reason)) 210 | return False 211 | 212 | def build_results_from_response(self): 213 | """ 214 | Build result from the response 215 | 216 | The reponse_from_server may contain up to 3 uri's. 217 | 1. link to fetch the next packet of data ('nextUri') 218 | 2. TODO: information about the query execution ('infoUri') 219 | 3. TODO: cancel the query ('partialCancelUri'). 220 | """ 221 | if NEXT_URI_RESP in self.response_from_server: 222 | self.next_uri = self.response_from_server[NEXT_URI_RESP] 223 | else: 224 | self.next_uri = '' 225 | 226 | if DATA_RESP in self.response_from_server: 227 | if self.rows: 228 | self.rows.extend(self.response_from_server[DATA_RESP]) 229 | else: 230 | self.rows = self.response_from_server[DATA_RESP] 231 | 232 | def get_rows(self, num_of_rows=NUM_ROWS): 233 | """ 234 | Get the rows returned from the query. 235 | 236 | The client sends GET requests to the server using the 'nextUri' 237 | from the previous response until the servers response does not 238 | contain anymore 'nextUri's. When there is no 'nextUri' the query is 239 | finished 240 | 241 | Note that this can only be called once and does not page through 242 | the results. 243 | 244 | Parameters: 245 | num_of_rows: to be retrieved. 1000 by default 246 | """ 247 | if num_of_rows == 0: 248 | return [] 249 | 250 | self.build_results_from_response() 251 | 252 | if not self.get_next_uri(): 253 | return [] 254 | 255 | while self.get_next_uri(): 256 | if not self.get_response_from(self.get_next_uri()): 257 | return [] 258 | if len(self.rows) <= num_of_rows: 259 | self.build_results_from_response() 260 | return self.rows 261 | 262 | def get_next_uri(self): 263 | return self.next_uri 264 | 265 | class InvalidArgumentError(ValueError): 266 | pass 267 | -------------------------------------------------------------------------------- /package/scripts/presto_coordinator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import uuid 16 | import os.path as path 17 | 18 | from resource_management.libraries.script.script import Script 19 | from resource_management.core.resources.system import Execute 20 | from resource_management.core.exceptions import ExecutionFailed, ComponentIsNotRunning 21 | from common import PRESTO_RPM_URL, PRESTO_RPM_NAME, create_connectors, \ 22 | delete_connectors 23 | from presto_client import smoketest_presto, PrestoClient 24 | 25 | 26 | class Coordinator(Script): 27 | def install(self, env): 28 | from params import java_home 29 | Execute('wget --no-check-certificate {0} -O /tmp/{1}'.format(PRESTO_RPM_URL, PRESTO_RPM_NAME)) 30 | Execute('export JAVA8_HOME={0} && rpm -i /tmp/{1}'.format(java_home, PRESTO_RPM_NAME)) 31 | self.configure(env) 32 | 33 | def stop(self, env): 34 | from params import daemon_control_script 35 | Execute('{0} stop'.format(daemon_control_script)) 36 | 37 | def start(self, env): 38 | from params import daemon_control_script, config_properties, \ 39 | host_info 40 | self.configure(env) 41 | Execute('{0} start'.format(daemon_control_script)) 42 | if 'presto_worker_hosts' in host_info.keys(): 43 | all_hosts = host_info['presto_worker_hosts'] + \ 44 | host_info['presto_coordinator_hosts'] 45 | else: 46 | all_hosts = host_info['presto_coordinator_hosts'] 47 | smoketest_presto(PrestoClient('localhost', 'root', config_properties['http-server.http.port']), all_hosts) 48 | 49 | def status(self, env): 50 | from params import daemon_control_script 51 | try: 52 | Execute('{0} status'.format(daemon_control_script)) 53 | except ExecutionFailed as ef: 54 | if ef.code == 3: 55 | raise ComponentIsNotRunning("ComponentIsNotRunning") 56 | else: 57 | raise ef 58 | 59 | def configure(self, env): 60 | from params import node_properties, jvm_config, config_properties, \ 61 | config_directory, memory_configs, host_info, connectors_to_add, connectors_to_delete 62 | key_val_template = '{0}={1}\n' 63 | 64 | with open(path.join(config_directory, 'node.properties'), 'w') as f: 65 | for key, value in node_properties.iteritems(): 66 | f.write(key_val_template.format(key, value)) 67 | f.write(key_val_template.format('node.id', str(uuid.uuid4()))) 68 | f.write(key_val_template.format('node.data-dir', '/var/lib/presto')) 69 | 70 | with open(path.join(config_directory, 'jvm.config'), 'w') as f: 71 | f.write(jvm_config['jvm.config']) 72 | 73 | with open(path.join(config_directory, 'config.properties'), 'w') as f: 74 | for key, value in config_properties.iteritems(): 75 | if key == 'query.queue-config-file' and value.strip() == '': 76 | continue 77 | if key in memory_configs: 78 | value += 'GB' 79 | f.write(key_val_template.format(key, value)) 80 | f.write(key_val_template.format('coordinator', 'true')) 81 | f.write(key_val_template.format('discovery-server.enabled', 'true')) 82 | 83 | create_connectors(node_properties, connectors_to_add) 84 | delete_connectors(node_properties, connectors_to_delete) 85 | # This is a separate call because we always want the tpch connector to 86 | # be available because it is used to smoketest the installation. 87 | create_connectors(node_properties, "{'tpch': ['connector.name=tpch']}") 88 | 89 | 90 | if __name__ == '__main__': 91 | Coordinator().execute() 92 | -------------------------------------------------------------------------------- /package/scripts/presto_worker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import uuid 16 | import os.path as path 17 | 18 | from resource_management.libraries.script.script import Script 19 | from resource_management.core.resources.system import Execute 20 | from resource_management.core.exceptions import ExecutionFailed, ComponentIsNotRunning 21 | from common import PRESTO_RPM_URL, PRESTO_RPM_NAME, create_connectors, \ 22 | delete_connectors 23 | 24 | 25 | class Worker(Script): 26 | def install(self, env): 27 | from params import java_home 28 | Execute('wget --no-check-certificate {0} -O /tmp/{1}'.format(PRESTO_RPM_URL, PRESTO_RPM_NAME)) 29 | Execute('export JAVA8_HOME={0} && rpm -i /tmp/{1}'.format(java_home, PRESTO_RPM_NAME)) 30 | self.configure(env) 31 | 32 | def stop(self, env): 33 | from params import daemon_control_script 34 | Execute('{0} stop'.format(daemon_control_script)) 35 | 36 | def start(self, env): 37 | from params import daemon_control_script 38 | self.configure(self) 39 | Execute('{0} start'.format(daemon_control_script)) 40 | 41 | def status(self, env): 42 | from params import daemon_control_script 43 | try: 44 | Execute('{0} status'.format(daemon_control_script)) 45 | except ExecutionFailed as ef: 46 | if ef.code == 3: 47 | raise ComponentIsNotRunning("ComponentIsNotRunning") 48 | else: 49 | raise ef 50 | 51 | 52 | def configure(self, env): 53 | from params import node_properties, jvm_config, config_properties, \ 54 | config_directory, memory_configs, connectors_to_add, connectors_to_delete 55 | key_val_template = '{0}={1}\n' 56 | 57 | with open(path.join(config_directory, 'node.properties'), 'w') as f: 58 | for key, value in node_properties.iteritems(): 59 | f.write(key_val_template.format(key, value)) 60 | f.write(key_val_template.format('node.id', str(uuid.uuid4()))) 61 | f.write(key_val_template.format('node.data-dir', '/var/lib/presto')) 62 | 63 | with open(path.join(config_directory, 'jvm.config'), 'w') as f: 64 | f.write(jvm_config['jvm.config']) 65 | 66 | with open(path.join(config_directory, 'config.properties'), 'w') as f: 67 | for key, value in config_properties.iteritems(): 68 | if key == 'query.queue-config-file' and value.strip() == '': 69 | continue 70 | if key in memory_configs: 71 | value += 'GB' 72 | f.write(key_val_template.format(key, value)) 73 | f.write(key_val_template.format('coordinator', 'false')) 74 | 75 | create_connectors(node_properties, connectors_to_add) 76 | delete_connectors(node_properties, connectors_to_delete) 77 | # This is a separate call because we always want the tpch connector to 78 | # be available because it is used to smoketest the installation. 79 | create_connectors(node_properties, "{'tpch': ['connector.name=tpch']}") 80 | 81 | if __name__ == '__main__': 82 | Worker().execute() 83 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tox==2.1.1 2 | mock==1.0.1 3 | virtualenv==12.0.7 4 | nose==1.3.7 5 | Sphinx==1.6.3 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | try: 17 | from setuptools import setup 18 | except ImportError: 19 | from distutils.core import setup 20 | 21 | with open('README.md') as readme_file: 22 | readme = readme_file.read() 23 | 24 | with open('HISTORY.rst') as history_file: 25 | history = history_file.read().replace('.. :changelog:', '') 26 | 27 | test_requirements = [ 28 | 'tox==1.9.2', 29 | 'nose==1.3.7', 30 | 'mock==1.0.1', 31 | ] 32 | 33 | 34 | setup( 35 | name='ambari-presto', 36 | version='1.3', 37 | description='This project contains the integration code for integrating \ 38 | Presto as a service in Ambari.', 39 | long_description=readme + '\n\n' + history, 40 | author='Teradata Corporation', 41 | author_email='anton.petrov@teradata.com', 42 | url='https://github.com/TeradataCenterForHadoop/ambari-presto-service', 43 | packages=['package.scripts'], 44 | include_package_data=True, 45 | license='APLv2', 46 | zip_safe=False, 47 | keywords=['presto', 'ambari', 'hadoop'], 48 | classifiers=[ 49 | 'Development Status :: 3 - Alpha', 50 | 'Intended Audience :: Developers', 51 | 'License :: OSI Approved :: APLv2 License', 52 | 'Natural Language :: English', 53 | "Programming Language :: Python :: 2", 54 | 'Programming Language :: Python :: 2.6', 55 | 'Programming Language :: Python :: 2.7' 56 | ], 57 | test_suite='tests', 58 | tests_require=test_requirements, 59 | ) 60 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/tests/__init__.py -------------------------------------------------------------------------------- /tests/resource_management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/tests/resource_management/__init__.py -------------------------------------------------------------------------------- /tests/resource_management/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/tests/resource_management/core/__init__.py -------------------------------------------------------------------------------- /tests/resource_management/core/exceptions.py: -------------------------------------------------------------------------------- 1 | class ClientComponentHasNoStatus(Exception): 2 | pass 3 | 4 | 5 | class ExecutionFailed(Exception): 6 | pass 7 | 8 | 9 | class ComponentIsNotRunning(Exception): 10 | pass 11 | -------------------------------------------------------------------------------- /tests/resource_management/core/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/tests/resource_management/core/resources/__init__.py -------------------------------------------------------------------------------- /tests/resource_management/core/resources/system.py: -------------------------------------------------------------------------------- 1 | def Execute(): 2 | pass 3 | -------------------------------------------------------------------------------- /tests/resource_management/libraries/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/tests/resource_management/libraries/__init__.py -------------------------------------------------------------------------------- /tests/resource_management/libraries/script/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/ambari-presto-service/51be4327dbd51bf3a1e8e40d05c2c2963de08766/tests/resource_management/libraries/script/__init__.py -------------------------------------------------------------------------------- /tests/resource_management/libraries/script/script.py: -------------------------------------------------------------------------------- 1 | class Script(object): 2 | 3 | def execute(self): 4 | pass 5 | 6 | @staticmethod 7 | def get_config(): 8 | return {'configurations': 9 | {'node.properties': {}, 10 | 'jvm.config': {'jvm.config': ''}, 11 | 'config.properties': {}, 12 | 'connectors.properties': {'connectors.to.add': '{}', 'connectors.to.delete': '{}'}}, 13 | 'clusterHostInfo': {'presto_worker_hosts': [], 'presto_coordinator_hosts': []}, 14 | 'hostLevelParams': {'java_home': '/some/fake/path'}} -------------------------------------------------------------------------------- /tests/resources/valid_rest_response_level1.txt: -------------------------------------------------------------------------------- 1 | { 2 | "id":"2015_harih", 3 | "infoUri":"http://localhost:8080/v1/query/2015_harih", 4 | "nextUri":"http://localhost:8080/v1/statement/2015_harih/2" 5 | } 6 | -------------------------------------------------------------------------------- /tests/resources/valid_rest_response_level2.txt: -------------------------------------------------------------------------------- 1 | { 2 | "id":"2015_harih", 3 | "infoUri":"http://localhost:8080/v1/query/2015_harih", 4 | "nextUri":"", 5 | "data":[["uuid1","http://localhost:8080","presto-main:0.97",true], ["uuid2","http://worker:8080","presto-main:0.97",false]] 6 | } 7 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mock import MagicMock, patch 16 | 17 | from unittest import TestCase 18 | import os 19 | import sys 20 | sys.path.append(os.path.dirname(os.path.realpath(__file__))) 21 | 22 | from package.scripts.presto_cli import Cli 23 | 24 | class TestCli(TestCase): 25 | 26 | def setUp(self): 27 | self.presto_cli = Cli() 28 | self.mock_env = MagicMock() 29 | 30 | @patch('package.scripts.presto_cli.Execute') 31 | def test_install_shells_out_to_execute(self, execute_mock): 32 | self.presto_cli.install(MagicMock) 33 | 34 | assert execute_mock.called 35 | 36 | # Raising ClientComponentHasNoStatus is needed so that restart 37 | # functionality works. 38 | def test_status_raises_exception(self): 39 | try: 40 | self.presto_cli.status(self.mock_env) 41 | except Exception as e: 42 | self.assertEqual(repr(e), 'ClientComponentHasNoStatus()') 43 | return 44 | TestCase.fail(self) 45 | 46 | # Client must implement status, configure, start and stop even 47 | # if those methods don't make sense for the component. They must 48 | # be implement for restart functionality to work correctly. 49 | def test_methods_overriden(self): 50 | assert 'configure' in dir(self.presto_cli) 51 | assert 'start' in dir(self.presto_cli) 52 | assert 'stop' in dir(self.presto_cli) 53 | assert 'status' in dir(self.presto_cli) -------------------------------------------------------------------------------- /tests/test_common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from unittest import TestCase 15 | from mock import patch, call 16 | import sys 17 | import os 18 | sys.path.append(os.path.dirname(os.path.realpath(__file__))) 19 | 20 | from test_worker import mock_file_descriptor_write_method 21 | 22 | 23 | class TestCommonCode(TestCase): 24 | node_properties = {'plugin.config-dir': '/does/not/exist'} 25 | 26 | @patch('package.scripts.common.Execute') 27 | def test_no_connectors_to_add(self, unused_execute_mock): 28 | from package.scripts.common import create_connectors 29 | 30 | connector_properties_written_out = collect_connector_properties_written_out( 31 | create_connectors, self.node_properties, '') 32 | 33 | assert not connector_properties_written_out 34 | 35 | 36 | connector_properties_written_out = collect_connector_properties_written_out( 37 | create_connectors, self.node_properties, '{}') 38 | 39 | assert not connector_properties_written_out 40 | 41 | @patch('package.scripts.common.Execute') 42 | def test_add_connector(self, unused_execute_mock): 43 | from package.scripts.common import create_connectors 44 | 45 | connector_properties = "{'hive': ['key1=value1', 'key2=value2']}" 46 | 47 | connector_properties_written_out = collect_connector_properties_written_out( 48 | create_connectors, self.node_properties, connector_properties) 49 | 50 | assert connector_properties_written_out == ['key1=value1\n', 'key2=value2\n'] 51 | 52 | @patch('package.scripts.common.Execute') 53 | def test_no_connectors_to_delete(self, execute_mock): 54 | from package.scripts.common import delete_connectors 55 | 56 | delete_connectors(self.node_properties, '') 57 | 58 | assert not execute_mock.called 59 | 60 | delete_connectors(self.node_properties, '{}') 61 | 62 | assert not execute_mock.called 63 | 64 | @patch('package.scripts.common.Execute') 65 | def test_delete_connector(self, execute_mock): 66 | from package.scripts.common import delete_connectors 67 | 68 | delete_connectors(self.node_properties, "['connector1', 'connector2']") 69 | 70 | assert execute_mock.call_args_list == [call('rm -f /does/not/exist/connector1.properties'), 71 | call('rm -f /does/not/exist/connector2.properties')] 72 | 73 | def collect_connector_properties_written_out( 74 | create_connectors_method, node_properties, connectors_properties): 75 | connector_properties_written_out = [] 76 | open_mock = mock_file_descriptor_write_method(connector_properties_written_out) 77 | 78 | with patch('__builtin__.open', open_mock): 79 | create_connectors_method(node_properties, connectors_properties) 80 | 81 | return connector_properties_written_out -------------------------------------------------------------------------------- /tests/test_coordinator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mock import MagicMock, patch, mock_open, call 16 | from unittest import TestCase 17 | 18 | import unittest 19 | import os 20 | import sys 21 | sys.path.append(os.path.dirname(os.path.realpath(__file__))) 22 | 23 | from package.scripts.presto_coordinator import Coordinator 24 | from package.scripts.params import memory_configs 25 | from test_worker import mock_file_descriptor_write_method, \ 26 | collect_config_vars_written_out 27 | 28 | 29 | class TestCoordinator(unittest.TestCase): 30 | 31 | dummy_config_properties = {'query.queue-config-file': '', 32 | 'http-server.http.port': '8285', 33 | 'node-scheduler.include-coordinator': 'true'} 34 | 35 | for memory_config in memory_configs: 36 | dummy_config_properties[memory_config] = '123' 37 | 38 | def setUp(self): 39 | self.mock_env = MagicMock() 40 | 41 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 42 | @patch('package.scripts.presto_coordinator.smoketest_presto') 43 | @patch('package.scripts.presto_coordinator.Coordinator.configure') 44 | @patch('package.scripts.presto_coordinator.Execute') 45 | def test_lifecycle_methods_shell_out_to_execute( 46 | self, execute_mock, unused_configure_mock, unused_smoketest_presto): 47 | presto_coordinator = Coordinator() 48 | 49 | presto_coordinator.install(self.mock_env) 50 | assert execute_mock.call_count is 2 51 | assert 'wget' in execute_mock.call_args_list[0][0][0] 52 | assert 'rpm -i' in execute_mock.call_args_list[1][0][0] 53 | assert 'export JAVA8_HOME=' in execute_mock.call_args_list[1][0][0] 54 | execute_mock.reset_mock() 55 | 56 | presto_coordinator.stop(self.mock_env) 57 | assert execute_mock.call_count is 1 58 | assert 'stop' in execute_mock.call_args_list[0][0][0] 59 | execute_mock.reset_mock() 60 | 61 | presto_coordinator.start(self.mock_env) 62 | assert execute_mock.call_count is 1 63 | assert 'start' in execute_mock.call_args_list[0][0][0] 64 | execute_mock.reset_mock() 65 | 66 | presto_coordinator.status(self.mock_env) 67 | assert execute_mock.call_count is 1 68 | assert 'status' in execute_mock.call_args_list[0][0][0] 69 | 70 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 71 | @patch('package.scripts.presto_coordinator.smoketest_presto') 72 | @patch('package.scripts.presto_coordinator.Coordinator.configure') 73 | @patch('package.scripts.presto_coordinator.Execute') 74 | def test_install_start_configure_presto( 75 | self, unused_execute_mock, configure_mock, unused_smoketest_presto): 76 | presto_coordinator = Coordinator() 77 | 78 | presto_coordinator.install(self.mock_env) 79 | assert configure_mock.called 80 | configure_mock.reset_mock() 81 | 82 | presto_coordinator.start(self.mock_env) 83 | assert configure_mock.called 84 | 85 | @patch('package.scripts.presto_coordinator.create_connectors') 86 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 87 | def test_configure_adds_tpch_connector(self, create_connectors_mock): 88 | presto_coordinator = Coordinator() 89 | open_mock = mock_open() 90 | 91 | with patch('__builtin__.open', open_mock): 92 | presto_coordinator.configure(self.mock_env) 93 | 94 | assert call({}, "{'tpch': ['connector.name=tpch']}") in create_connectors_mock.call_args_list 95 | 96 | 97 | @patch('package.scripts.presto_coordinator.create_connectors') 98 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 99 | @patch('package.scripts.presto_coordinator.smoketest_presto') 100 | @patch('package.scripts.presto_coordinator.Coordinator.configure') 101 | @patch('package.scripts.presto_coordinator.Execute') 102 | def test_start_smoketests_presto( 103 | self, execute_mock, unused_configure_mock, smoketest_presto_mock, create_connectors_mock): 104 | presto_coordinator = Coordinator() 105 | 106 | presto_coordinator.start(self.mock_env) 107 | 108 | assert smoketest_presto_mock.called 109 | 110 | @patch('package.scripts.presto_coordinator.create_connectors') 111 | @patch('package.scripts.params.config_properties', new={}) 112 | def test_assert_constant_properties(self, create_connectors_mock): 113 | config = collect_config_vars_written_out(self.mock_env, Coordinator()) 114 | 115 | assert 'discovery-server.enabled=true\n' in config 116 | assert 'coordinator=true\n' in config 117 | assert 'node.data-dir=/var/lib/presto\n' 118 | 119 | @patch('package.scripts.presto_coordinator.create_connectors') 120 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 121 | def test_configure_ignore_empty_queue_config_file(self, create_connectors_mock): 122 | config = collect_config_vars_written_out(self.mock_env, Coordinator()) 123 | 124 | for item in config: 125 | assert not item.startswith('query.queue-config-file') 126 | 127 | @patch('package.scripts.params.host_info', new={'presto_coordinator_hosts': ['master']}) 128 | @patch('package.scripts.presto_coordinator.create_connectors') 129 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 130 | def test_configure_pseudo_distributed(self, create_connectors_mock): 131 | config = collect_config_vars_written_out(self.mock_env, Coordinator()) 132 | 133 | assert 'node-scheduler.include-coordinator=true\n' in config 134 | 135 | @patch('package.scripts.presto_coordinator.create_connectors') 136 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 137 | def test_memory_settings_have_units(self, create_connectos_mock): 138 | config = collect_config_vars_written_out(self.mock_env, Coordinator()) 139 | 140 | assert_memory_configs_properly_formatted(config) 141 | 142 | def assert_memory_configs_properly_formatted(configs_to_test): 143 | import re 144 | from package.scripts.params import memory_configs 145 | 146 | for memory_config in memory_configs: 147 | result = [x for x in configs_to_test \ 148 | if re.match(memory_config + '=\d*GB\n', x)] 149 | assert len(result) == 1 150 | -------------------------------------------------------------------------------- /tests/test_presto_client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import re 16 | import socket 17 | import os 18 | import json 19 | import sys 20 | sys.path.append(os.path.dirname(os.path.realpath(__file__))) 21 | 22 | from mock import MagicMock, patch, PropertyMock 23 | from httplib import HTTPException, HTTPConnection 24 | from unittest import TestCase 25 | 26 | from package.scripts.presto_client import smoketest_presto, PrestoClient, \ 27 | InvalidArgumentError, URL_TIMEOUT_MS 28 | 29 | class TestPrestoClientSmoketest(TestCase): 30 | 31 | nation = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 32 | 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] 33 | 34 | def setUp(self): 35 | self.presto_client_mock = MagicMock() 36 | 37 | @patch('package.scripts.presto_client.time.sleep', return_value=None) 38 | def test_presto_queried_on_smoketest(self, unused_sleep_mock): 39 | self.presto_client_mock.get_rows.side_effect = [[['master']], 'dummy_val', TestPrestoClientSmoketest.nation] 40 | 41 | smoketest_presto(self.presto_client_mock, ['master']) 42 | 43 | assert self.presto_client_mock.execute_query.called 44 | 45 | @patch('package.scripts.presto_client.time.sleep', return_value=None) 46 | def test_failure_when_fewer_than_25_rows_returned_from_nation(self, unused_sleep_mock): 47 | self.presto_client_mock.get_rows.return_value = [['master']] 48 | 49 | TestCase.assertRaises(self, RuntimeError, smoketest_presto, self.presto_client_mock, ['master']) 50 | 51 | @patch('package.scripts.presto_client.ensure_nodes_are_up', side_effect=RuntimeError()) 52 | def test_failure_when_nodes_are_not_up(self, ensure_nodes_are_up_mock): 53 | TestCase.assertRaises(self, RuntimeError, smoketest_presto, self.presto_client_mock, ['master']) 54 | 55 | @patch('package.scripts.presto_client.time.sleep', return_value=None) 56 | @patch('package.scripts.presto_client.ensure_catalogs_are_available', side_effect=RuntimeError()) 57 | def test_failure_when_catalogs_are_not_available(self, ensure_catalogs_are_available_mock, 58 | unused_sleep_mock): 59 | TestCase.assertRaises(self, RuntimeError, smoketest_presto, self.presto_client_mock, ['master']) 60 | 61 | @patch('package.scripts.presto_client.time.sleep', return_value=None) 62 | def test_failure_when_nodes_returned_dont_match_nodes_specified(self, unused_sleep_mock): 63 | self.presto_client_mock.get_rows.return_value = [['bad_host']] 64 | 65 | TestCase.assertRaises(self, RuntimeError, smoketest_presto, self.presto_client_mock, ['master']) 66 | 67 | # These tests were copied more or less verbatim from 68 | # https://github.com/prestodb/presto-admin/blob/master/tests/unit/test_prestoclient.py 69 | class TestPrestoClient(TestCase): 70 | 71 | def test_no_sql(self): 72 | client = PrestoClient('any_host', 'any_user', 8080) 73 | self.assertRaisesRegexp(InvalidArgumentError, 74 | 'SQL query missing', 75 | client.execute_query, '', ) 76 | 77 | def test_no_server(self): 78 | client = PrestoClient('', 'any_user', 8080) 79 | self.assertRaisesRegexp(InvalidArgumentError, 80 | 'Server IP missing', 81 | client.execute_query, 'any_sql') 82 | 83 | def test_no_user(self): 84 | client = PrestoClient('any_host', '', 8080) 85 | self.assertRaisesRegexp(InvalidArgumentError, 86 | 'Username missing', 87 | client.execute_query, 'any_sql') 88 | 89 | @patch('package.scripts.presto_client.HTTPConnection') 90 | def test_default_request_called(self, mock_conn): 91 | client = PrestoClient('any_host', 'any_user', 8080) 92 | headers = {'X-Presto-Catalog': 'tpch', 'X-Presto-Schema': 'sf1', 93 | 'X-Presto-User': 'any_user'} 94 | 95 | client.execute_query('any_sql') 96 | mock_conn.assert_called_with('any_host', 8080, False, URL_TIMEOUT_MS) 97 | mock_conn().request.assert_called_with('POST', '/v1/statement', 98 | 'any_sql', headers) 99 | self.assertTrue(mock_conn().getresponse.called) 100 | 101 | @patch('package.scripts.presto_client.HTTPConnection') 102 | def test_connection_failed(self, mock_conn): 103 | client = PrestoClient('any_host', 'any_user', 8080) 104 | client.execute_query('any_sql') 105 | 106 | self.assertTrue(mock_conn().close.called) 107 | self.assertFalse(client.execute_query('any_sql')) 108 | 109 | @patch('package.scripts.presto_client.HTTPConnection') 110 | def test_http_call_failed(self, mock_conn): 111 | client = PrestoClient('any_host', 'any_user', 8080) 112 | mock_conn.side_effect = HTTPException('Error') 113 | self.assertFalse(client.execute_query('any_sql')) 114 | 115 | mock_conn.side_effect = socket.error('Error') 116 | self.assertFalse(client.execute_query('any_sql')) 117 | 118 | @patch.object(HTTPConnection, 'request') 119 | @patch.object(HTTPConnection, 'getresponse') 120 | def test_http_answer_valid(self, mock_response, mock_request): 121 | client = PrestoClient('any_host', 'any_user', 8080) 122 | mock_response.return_value.read.return_value = '{}' 123 | type(mock_response.return_value).status = \ 124 | PropertyMock(return_value=200) 125 | self.assertTrue(client.execute_query('any_sql')) 126 | 127 | @patch.object(HTTPConnection, 'request') 128 | @patch.object(HTTPConnection, 'getresponse') 129 | def test_http_answer_not_json(self, mock_response, mock_request): 130 | client = PrestoClient('any_host', 'any_user', 8080) 131 | mock_response.return_value.read.return_value = 'NOT JSON!' 132 | type(mock_response.return_value).status =\ 133 | PropertyMock(return_value=200) 134 | self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', 135 | client.execute_query, 'any_sql') 136 | 137 | @patch.object(PrestoClient, 'get_response_from') 138 | @patch.object(PrestoClient, 'get_next_uri') 139 | def test_retrieve_rows(self, mock_uri, mock_get_from_uri): 140 | client = PrestoClient('any_host', 'any_user', 8080) 141 | dir = os.path.abspath(os.path.dirname(__file__)) 142 | 143 | with open(dir + '/resources/valid_rest_response_level1.txt') \ 144 | as json_file: 145 | client.response_from_server = json.load(json_file) 146 | mock_get_from_uri.return_value = True 147 | mock_uri.side_effect = [ 148 | "http://localhost:8080/v1/statement/2015_harih/2", "" 149 | ] 150 | 151 | self.assertEqual(client.get_rows(), []) 152 | self.assertEqual(client.next_uri, 153 | "http://localhost:8080/v1/statement/2015_harih/2") 154 | 155 | with open(dir + '/resources/valid_rest_response_level2.txt') \ 156 | as json_file: 157 | client.response_from_server = json.load(json_file) 158 | mock_uri.side_effect = [ 159 | "http://localhost:8080/v1/statement/2015_harih/2", "" 160 | ] 161 | 162 | expected_row = [["uuid1", "http://localhost:8080", "presto-main:0.97", 163 | True], 164 | ["uuid2", "http://worker:8080", "presto-main:0.97", 165 | False]] 166 | self.assertEqual(client.get_rows(), expected_row) 167 | self.assertEqual(client.next_uri, "") 168 | 169 | @patch.object(PrestoClient, 'get_response_from') 170 | @patch.object(PrestoClient, 'get_next_uri') 171 | def test_append_rows(self, mock_uri, mock_get_from_uri): 172 | client = PrestoClient('any_host', 'any_user', 8080) 173 | dir = os.path.abspath(os.path.dirname(__file__)) 174 | 175 | with open(dir + '/resources/valid_rest_response_level2.txt') \ 176 | as json_file: 177 | client.response_from_server = json.load(json_file) 178 | mock_get_from_uri.return_value = True 179 | mock_uri.side_effect = ["any_next_uri", "any_next_next_uri", "", ""] 180 | expected_row = [["uuid1", "http://localhost:8080", "presto-main:0.97", 181 | True], 182 | ["uuid2", "http://worker:8080", "presto-main:0.97", 183 | False], 184 | ["uuid1", "http://localhost:8080", "presto-main:0.97", 185 | True], 186 | ["uuid2", "http://worker:8080", "presto-main:0.97", 187 | False]] 188 | self.assertEqual(client.get_rows(), expected_row) 189 | 190 | @patch.object(PrestoClient, 'get_response_from') 191 | @patch.object(PrestoClient, 'get_next_uri') 192 | def test_limit_rows(self, mock_uri, mock_get_from_uri): 193 | client = PrestoClient('any_host', 'any_user', 8080) 194 | dir = os.path.abspath(os.path.dirname(__file__)) 195 | with open(dir + '/resources/valid_rest_response_level2.txt') \ 196 | as json_file: 197 | client.response_from_server = json.load(json_file) 198 | mock_get_from_uri.return_value = True 199 | mock_uri.side_effect = ["any_next_uri", ""] 200 | 201 | self.assertEqual(client.get_rows(0), []) 202 | 203 | @patch('package.scripts.presto_client.urlopen') 204 | @patch('httplib.HTTPResponse') 205 | def test_get_response(self, mock_resp, mock_urlopen): 206 | client = PrestoClient('any_host', 'any_user', 8080) 207 | mock_urlopen.return_value = mock_resp 208 | mock_resp.read.return_value = '{"message": "ok!"}' 209 | 210 | client.get_response_from('any_uri') 211 | self.assertEqual(client.response_from_server, {"message": "ok!"}) 212 | 213 | # This method is equivalent to Python 2.7's unittest.assertRaisesRegexp() 214 | def assertRaisesRegexp(self, expected_exception, expected_regexp, 215 | callable_object, *args, **kwargs): 216 | if 'msg' in kwargs and kwargs['msg']: 217 | msg = '\n' + kwargs['msg'] 218 | else: 219 | msg = '' 220 | try: 221 | callable_object(*args) 222 | except expected_exception as e: 223 | self.assertTrue(re.search(expected_regexp, str(e)), 224 | repr(expected_regexp) + ' not found in ' 225 | + repr(str(e)) + msg) 226 | else: 227 | self.fail('Expected exception ' + str(expected_exception) + 228 | ' not raised' + msg) -------------------------------------------------------------------------------- /tests/test_worker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mock import MagicMock, patch, mock_open, call 16 | 17 | import unittest 18 | import os 19 | import sys 20 | sys.path.append(os.path.dirname(os.path.realpath(__file__))) 21 | 22 | from package.scripts.presto_worker import Worker 23 | from package.scripts.params import memory_configs 24 | 25 | 26 | class TestWorker(unittest.TestCase): 27 | 28 | dummy_config_properties = {'pseudo.distributed.enabled': False, 29 | 'query.queue-config-file': '', 30 | 'http-server.http.port': '8285', 31 | 'node-scheduler.include-coordinator': False} 32 | 33 | minimal_config_properties = {'pseudo.distributed.enabled': False, 34 | 'node-scheduler.include-coordinator': False} 35 | 36 | for memory_config in memory_configs: 37 | dummy_config_properties[memory_config] = '123' 38 | 39 | def setUp(self): 40 | self.mock_env = MagicMock() 41 | 42 | @patch('package.scripts.presto_worker.Worker.configure') 43 | @patch('package.scripts.presto_worker.Execute') 44 | def test_lifecycle_methods_shell_out_to_execute( 45 | self, execute_mock, unused_configure_mock): 46 | presto_worker = Worker() 47 | 48 | presto_worker.install(self.mock_env) 49 | assert execute_mock.call_count is 2 50 | assert 'wget' in execute_mock.call_args_list[0][0][0] 51 | assert 'rpm -i' in execute_mock.call_args_list[1][0][0] 52 | assert 'export JAVA8_HOME=' in execute_mock.call_args_list[1][0][0] 53 | execute_mock.reset_mock() 54 | 55 | presto_worker.stop(self.mock_env) 56 | assert execute_mock.call_count is 1 57 | assert 'stop' in execute_mock.call_args_list[0][0][0] 58 | execute_mock.reset_mock() 59 | 60 | presto_worker.start(self.mock_env) 61 | assert execute_mock.call_count is 1 62 | assert 'start' in execute_mock.call_args_list[0][0][0] 63 | execute_mock.reset_mock() 64 | 65 | presto_worker.status(self.mock_env) 66 | assert execute_mock.call_count is 1 67 | assert 'status' in execute_mock.call_args_list[0][0][0] 68 | 69 | @patch('package.scripts.presto_worker.Worker.configure') 70 | @patch('package.scripts.presto_worker.Execute') 71 | def test_install_start_configure_presto( 72 | self, unused_execute_mock, configure_mock): 73 | presto_worker = Worker() 74 | 75 | presto_worker.install(self.mock_env) 76 | assert configure_mock.called 77 | configure_mock.reset_mock() 78 | 79 | presto_worker.start(self.mock_env) 80 | assert configure_mock.called 81 | 82 | @patch('package.scripts.presto_worker.create_connectors') 83 | def test_configure_adds_tpch_connector(self, create_connectors_mock): 84 | presto_worker = Worker() 85 | 86 | with patch('__builtin__.open'): 87 | presto_worker.configure(self.mock_env) 88 | 89 | assert call({}, "{'tpch': ['connector.name=tpch']}") in create_connectors_mock.call_args_list 90 | 91 | @patch('package.scripts.presto_worker.create_connectors') 92 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 93 | def test_configure_ignore_pseudo_distribute_enabled_property(self, create_connectors_mock ): 94 | config = collect_config_vars_written_out(self.mock_env, Worker()) 95 | 96 | assert 'pseudo.distributed.enabled=true\n' not in config 97 | 98 | @patch('package.scripts.presto_worker.create_connectors') 99 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 100 | def test_configure_ignore_empty_queue_config_file(self, create_connectors_mock): 101 | config = collect_config_vars_written_out(self.mock_env, Worker()) 102 | 103 | for item in config: 104 | assert not item.startswith('query.queue-config-file') 105 | 106 | @patch('package.scripts.presto_worker.create_connectors') 107 | @patch('package.scripts.params.config_properties', new=minimal_config_properties) 108 | def test_constant_properties(self, create_connectors_mock): 109 | config = collect_config_vars_written_out(self.mock_env, Worker()) 110 | 111 | assert 'coordinator=false\n' in config 112 | assert 'node.data-dir=/var/lib/presto\n' in config 113 | 114 | @patch('package.scripts.presto_worker.create_connectors') 115 | @patch('package.scripts.params.config_properties', new=dummy_config_properties) 116 | def test_memory_settings_have_units(self, create_connectors_mock): 117 | from test_coordinator import assert_memory_configs_properly_formatted 118 | 119 | config = collect_config_vars_written_out(self.mock_env, Worker()) 120 | assert_memory_configs_properly_formatted(config) 121 | 122 | 123 | def collect_config_vars_written_out(mock_env, obj_under_test): 124 | config = [] 125 | open_mock = mock_file_descriptor_write_method(config) 126 | 127 | with patch('__builtin__.open', open_mock): 128 | getattr(obj_under_test, 'configure')(mock_env) 129 | 130 | return config 131 | 132 | def mock_file_descriptor_write_method(list): 133 | def append(item_to_append): 134 | list.append(item_to_append) 135 | 136 | open_mock = mock_open() 137 | fd = open_mock() 138 | fd.write = append 139 | return open_mock 140 | -------------------------------------------------------------------------------- /themes/theme.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "default", 3 | "description": "Default theme for Presto service", 4 | "configuration": { 5 | "layouts": [ 6 | { 7 | "name": "default", 8 | "tabs": [ 9 | { 10 | "name": "settings", 11 | "display-name": "Settings", 12 | "layout": { 13 | "tab-columns": "2", 14 | "tab-rows": "3", 15 | "sections": [ 16 | { 17 | "name": "section-node-config", 18 | "display-name": "Node Config", 19 | "row-index": "0", 20 | "column-index": "0", 21 | "row-span": "1", 22 | "column-span": "1", 23 | "subsections": [ 24 | { 25 | "name": "subsection-node-config", 26 | "row-index": "0", 27 | "column-index": "0", 28 | "row-span": "1", 29 | "column-span": "1" 30 | } 31 | ] 32 | }, 33 | { 34 | "name": "section-general-config", 35 | "display-name": "General Config", 36 | "row-index": "0", 37 | "column-index": "1", 38 | "row-span": "3", 39 | "column-span": "1", 40 | "subsections": [ 41 | { 42 | "name": "subsection-general-config", 43 | "row-index": "0", 44 | "column-index": "0", 45 | "row-span": "3", 46 | "column-span": "1" 47 | } 48 | ] 49 | }, 50 | { 51 | "name": "section-connector-config", 52 | "display-name": "Connectors", 53 | "row-index": "1", 54 | "column-index": "0", 55 | "row-span": "1", 56 | "column-span": "1", 57 | "subsections": [ 58 | { 59 | "name": "subsection-connector-config", 60 | "row-index": "0", 61 | "column-index": "0", 62 | "row-span": "1", 63 | "column-span": "1" 64 | } 65 | ] 66 | }, 67 | { 68 | "name": "section-jvm-config", 69 | "display-name": "JVM Config", 70 | "row-index": "2", 71 | "column-index": "0", 72 | "row-span": "1", 73 | "column-span": "1", 74 | "subsections": [ 75 | { 76 | "name": "subsection-jvm-config", 77 | "row-index": "0", 78 | "column-index": "0", 79 | "row-span": "1", 80 | "column-span": "1" 81 | } 82 | ] 83 | } 84 | ] 85 | } 86 | } 87 | ] 88 | } 89 | ], 90 | "placement": { 91 | "configuration-layout": "default", 92 | "configs": [ 93 | { 94 | "config": "node.properties/node.environment", 95 | "subsection-name": "subsection-node-config" 96 | }, 97 | { 98 | "config": "node.properties/plugin.config-dir", 99 | "subsection-name": "subsection-node-config" 100 | }, 101 | { 102 | "config": "node.properties/plugin.dir", 103 | "subsection-name": "subsection-node-config" 104 | }, 105 | { 106 | "config": "connectors.properties/connectors.to.add", 107 | "subsection-name": "subsection-connector-config" 108 | }, 109 | { 110 | "config": "connectors.properties/connectors.to.delete", 111 | "subsection-name": "subsection-connector-config" 112 | }, 113 | { 114 | "config": "config.properties/node-scheduler.include-coordinator", 115 | "subsection-name": "subsection-general-config" 116 | }, 117 | { 118 | "config": "config.properties/http-server.http.port", 119 | "subsection-name": "subsection-general-config" 120 | }, 121 | { 122 | "config": "config.properties/query.max-memory", 123 | "subsection-name": "subsection-general-config" 124 | }, 125 | { 126 | "config": "config.properties/query.max-memory-per-node", 127 | "subsection-name": "subsection-general-config" 128 | }, 129 | { 130 | "config": "config.properties/discovery.uri", 131 | "subsection-name": "subsection-general-config" 132 | }, 133 | { 134 | "config": "jvm.config/jvm.config", 135 | "subsection-name": "subsection-jvm-config" 136 | } 137 | ] 138 | }, 139 | "widgets": [ 140 | { 141 | "config": "node.properties/node.environment", 142 | "widget": { 143 | "type": "text-area" 144 | } 145 | }, 146 | { 147 | "config": "node.properties/plugin.config-dir", 148 | "widget": { 149 | "type": "directory" 150 | } 151 | }, 152 | { 153 | "config": "node.properties/plugin.dir", 154 | "widget": { 155 | "type": "directory" 156 | } 157 | }, 158 | { 159 | "config": "connectors.properties/connectors.to.add", 160 | "widget": { 161 | "type": "text-area" 162 | } 163 | }, 164 | { 165 | "config": "connectors.properties/connectors.to.delete", 166 | "widget": { 167 | "type": "text-area" 168 | } 169 | }, 170 | { 171 | "config": "config.properties/node-scheduler.include-coordinator", 172 | "widget": { 173 | "type": "toggle" 174 | } 175 | }, 176 | { 177 | "config": "config.properties/http-server.http.port", 178 | "widget": { 179 | "type": "text-area" 180 | } 181 | }, 182 | { 183 | "config": "config.properties/query.max-memory", 184 | "widget": { 185 | "type": "slider", 186 | "units": [ 187 | { 188 | "unit-name": "GB" 189 | } 190 | ] 191 | } 192 | }, 193 | { 194 | "config": "config.properties/query.max-memory-per-node", 195 | "widget": { 196 | "type": "slider", 197 | "units": [ 198 | { 199 | "unit-name": "GB" 200 | } 201 | ] 202 | } 203 | }, 204 | { 205 | "config": "config.properties/discovery.uri", 206 | "widget": { 207 | "type": "text-area" 208 | } 209 | }, 210 | { 211 | "config": "jvm.config/jvm.config", 212 | "widget": { 213 | "type": "text-area" 214 | } 215 | } 216 | ] 217 | } 218 | } -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27 3 | 4 | [testenv] 5 | setenv = PYTHONPATH = {toxinidir}:{toxinidir}/package 6 | deps = 7 | -r{toxinidir}/requirements.txt 8 | commands = nosetests {posargs} --------------------------------------------------------------------------------