├── .bumpversion.cfg ├── .editorconfig ├── .gitignore ├── .readthedocs.yaml ├── .travis.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── docker_drill_embedded.sh ├── docs ├── Makefile ├── authors.rst ├── conf.py ├── contributing.rst ├── history.rst ├── index.rst ├── installation.rst ├── make.bat ├── readme.rst ├── requirements.txt └── usage.rst ├── pydrill ├── __init__.py ├── client │ ├── __init__.py │ └── result.py ├── compat.py ├── connection │ ├── __init__.py │ ├── base.py │ └── requests_conn.py ├── exceptions.py ├── serializer.py └── transport.py ├── requirements_base.txt ├── requirements_dev.txt ├── requirements_testing.txt ├── run_docker.sh ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── test_authentication.py ├── test_metrics.py ├── test_options.py ├── test_profiles.py ├── test_pydrill_setup.py ├── test_query.py ├── test_storage.py └── test_threads.py ├── tox.ini └── travis_pypi_setup.py /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.3.4 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | 8 | [bumpversion:file:pydrill/__init__.py] 9 | 10 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # File: .readthedocs.yaml 2 | version: 2 3 | 4 | sphinx: 5 | configuration: docs/conf.py 6 | 7 | python: 8 | version: 3.7 9 | install: 10 | - requirements: docs/requirements.txt 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | services: 2 | - docker 3 | sudo: required 4 | addons: 5 | apt: 6 | packages: 7 | - python3.6 8 | - python3.5 9 | - python3.5-dev 10 | sources: 11 | - deadsnakes 12 | after_success: 13 | - coveralls 14 | before_install: 15 | - docker ps -a 16 | - source ./run_docker.sh 17 | - docker ps -a 18 | env: 19 | - TOXENV=check-isort 20 | - TOXENV=check-flake8 21 | - TOXENV=python3.6 22 | - TOXENV=python3.5 23 | - TOXENV=python3.4 24 | - TOXENV=python3.3 25 | - TOXENV=python2.7 26 | - TOXENV=python2.6 27 | install: 28 | - pip install -U tox 29 | - pip install coveralls 30 | language: python 31 | script: 32 | - tox 33 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Wojciech Nowak 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every 8 | little bit helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/PythonicNinja/pydrill/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | Fix Bugs 27 | ~~~~~~~~ 28 | 29 | Look through the GitHub issues for bugs. Anything tagged with "bug" 30 | is open to whoever wants to implement it. 31 | 32 | Implement Features 33 | ~~~~~~~~~~~~~~~~~~ 34 | 35 | Look through the GitHub issues for features. Anything tagged with "feature" 36 | is open to whoever wants to implement it. 37 | 38 | Write Documentation 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | pydrill could always use more documentation, whether as part of the 42 | official pydrill docs, in docstrings, or even on the web in blog posts, 43 | articles, and such. 44 | 45 | Submit Feedback 46 | ~~~~~~~~~~~~~~~ 47 | 48 | The best way to send feedback is to file an issue at https://github.com/PythonicNinja/pydrill/issues. 49 | 50 | If you are proposing a feature: 51 | 52 | * Explain in detail how it would work. 53 | * Keep the scope as narrow as possible, to make it easier to implement. 54 | * Remember that this is a volunteer-driven project, and that contributions 55 | are welcome :) 56 | 57 | Get Started! 58 | ------------ 59 | 60 | Ready to contribute? Here's how to set up `pydrill` for local development. 61 | 62 | 1. Fork the `pydrill` repo on GitHub. 63 | 2. Clone your fork locally:: 64 | 65 | $ git clone git@github.com:your_name_here/pydrill.git 66 | 67 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 68 | 69 | $ mkvirtualenv pydrill 70 | $ cd pydrill/ 71 | $ python setup.py develop 72 | 73 | 4. Create a branch for local development:: 74 | 75 | $ git checkout -b name-of-your-bugfix-or-feature 76 | 77 | Now you can make your changes locally. 78 | 79 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: 80 | 81 | $ source ./run_docker (it will export PYDRILL_HOST and PYDRILL_PORT) 82 | or 83 | $ export PYDRILL_HOST='127.0.0.1' 84 | $ export PYDRILL_PORT='8047' 85 | $ tox -e run-isort # it will update imports 86 | $ tox -e check-isort # it will check if imports are correct 87 | $ tox -e check-flake8 # it will check quality by flake8 88 | $ tox -e py27 # run tests with py27 89 | $ tox # run all 90 | 91 | To get flake8 and tox, just pip install them into your virtualenv. 92 | 93 | 6. Commit your changes and push your branch to GitHub:: 94 | 95 | $ git add . 96 | $ git commit -m "Your detailed description of your changes." 97 | $ git push origin name-of-your-bugfix-or-feature 98 | 99 | 7. Submit a pull request through the GitHub website. 100 | 101 | Pull Request Guidelines 102 | ----------------------- 103 | 104 | Before you submit a pull request, check that it meets these guidelines: 105 | 106 | 1. The pull request should include tests. 107 | 2. If the pull request adds functionality, the docs should be updated. Put 108 | your new functionality into a function with a docstring, and add the 109 | feature to the list in README.rst. 110 | 3. The pull request should work for Python 2.6, 2.7, 3.3, 3.4 and 3.5, and for PyPy. Check 111 | https://travis-ci.org/PythonicNinja/pydrill/pull_requests 112 | and make sure that the tests pass for all supported Python versions. 113 | 114 | Tips 115 | ---- 116 | 117 | To run a subset of tests:: 118 | 119 | $ tox -e py27 -- -k threads # it will only run tests which have key threads in name 120 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | 0.3.4 (2017-04-24) 6 | ------------------ 7 | * Updated pypi listing long_description 8 | 9 | 0.3.3 (2017-04-24) 10 | ------------------ 11 | * Fix pypi installation 12 | 13 | 0.3.2 (2017-04-18) 14 | ------------------ 15 | * Support for dtype on to_dataframe 16 | 17 | 0.3.1 (2017-03-06) 18 | ------------------ 19 | * Support for Drill Authentication using PAM 20 | 21 | 0.3 (2017-02-15) 22 | ---------------- 23 | * requests response encoding (utf-8) 24 | * support Python 3.6 support 25 | 26 | 0.1.1 (2016-05-21) 27 | ------------------ 28 | * Anaconda requirements fixed 29 | 30 | 0.1.0 (2016-05-19) 31 | ------------------ 32 | * First minor release 33 | * Updated docs 34 | 35 | 0.0.2 (2016-04-24) 36 | ------------------ 37 | * First release on PyPI. 38 | * Implementation of metrics/storage/options/stats 39 | * Builds are tested by docker container with Apache Drill running 40 | * support for pandas with ResultQuery.to_dataframe 41 | 42 | 0.0.1 (2015-12-28) 43 | ------------------ 44 | * Project start 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Wojtek Nowak 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | include requirements_base.txt 7 | include requirements_testing.txt 8 | 9 | recursive-include tests * 10 | recursive-exclude * __pycache__ 11 | recursive-exclude * *.py[co] 12 | 13 | recursive-include docs *.rst conf.py Makefile make.bat 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs clean 2 | define BROWSER_PYSCRIPT 3 | import os, webbrowser, sys 4 | try: 5 | from urllib import pathname2url 6 | except: 7 | from urllib.request import pathname2url 8 | 9 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 10 | endef 11 | export BROWSER_PYSCRIPT 12 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 13 | 14 | help: 15 | @echo "clean - remove all build, test, coverage and Python artifacts" 16 | @echo "clean-build - remove build artifacts" 17 | @echo "clean-pyc - remove Python file artifacts" 18 | @echo "clean-test - remove test and coverage artifacts" 19 | @echo "lint - check style with flake8" 20 | @echo "test - run tests quickly with the default Python" 21 | @echo "test-all - run tests on every Python version with tox" 22 | @echo "coverage - check code coverage quickly with the default Python" 23 | @echo "docs - generate Sphinx HTML documentation, including API docs" 24 | @echo "release - package and upload a release" 25 | @echo "dist - package" 26 | @echo "install - install the package to the active Python's site-packages" 27 | 28 | clean: clean-build clean-pyc clean-test 29 | 30 | clean-build: 31 | rm -fr build/ 32 | rm -fr dist/ 33 | rm -fr .eggs/ 34 | find . -name '*.egg-info' -exec rm -fr {} + 35 | find . -name '*.egg' -exec rm -f {} + 36 | 37 | clean-pyc: 38 | find . -name '*.pyc' -exec rm -f {} + 39 | find . -name '*.pyo' -exec rm -f {} + 40 | find . -name '*~' -exec rm -f {} + 41 | find . -name '__pycache__' -exec rm -fr {} + 42 | 43 | clean-test: 44 | rm -fr .tox/ 45 | rm -f .coverage 46 | rm -fr htmlcov/ 47 | 48 | lint: 49 | flake8 pydrill tests 50 | 51 | test: 52 | python setup.py test 53 | 54 | test-all: 55 | tox 56 | 57 | coverage: 58 | coverage run --source pydrill setup.py test 59 | coverage report -m 60 | coverage html 61 | $(BROWSER) htmlcov/index.html 62 | 63 | docs: 64 | rm -f docs/pydrill.rst 65 | rm -f docs/modules.rst 66 | sphinx-apidoc -o docs/ pydrill 67 | $(MAKE) -C docs clean 68 | $(MAKE) -C docs html 69 | $(BROWSER) docs/_build/html/index.html 70 | 71 | servedocs: docs 72 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . 73 | 74 | release: clean 75 | python setup.py sdist upload 76 | python setup.py bdist_wheel upload 77 | 78 | dist: clean 79 | python setup.py sdist 80 | python setup.py bdist_wheel 81 | ls -l dist 82 | 83 | install: clean 84 | python setup.py install 85 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | pydrill 3 | =============================== 4 | 5 | .. image:: https://img.shields.io/travis/PythonicNinja/pydrill.svg 6 | :target: https://travis-ci.org/PythonicNinja/pydrill 7 | 8 | .. image:: https://img.shields.io/pypi/v/pydrill.svg 9 | :target: https://pypi.python.org/pypi/pydrill 10 | 11 | .. image:: https://readthedocs.org/projects/pydrill/badge/?version=latest 12 | :target: https://readthedocs.org/projects/pydrill/?badge=latest 13 | :alt: Documentation Status 14 | 15 | .. image:: https://coveralls.io/repos/github/PythonicNinja/pydrill/badge.svg?branch=master 16 | :target: https://coveralls.io/github/PythonicNinja/pydrill?branch=master 17 | 18 | 19 | Python Driver for `Apache Drill `_. 20 | 21 | *Schema-free SQL Query Engine for Hadoop, NoSQL and Cloud Storage* 22 | 23 | * Free software: MIT license 24 | * Documentation: https://pydrill.readthedocs.org. 25 | 26 | Features 27 | -------- 28 | 29 | * Python 2/3 compatibility, 30 | * Support for all rest API calls inluding profiles/options/metrics `docs with full list `_. 31 | * Mapping Results to internal python types, 32 | * Compatibility with Pandas data frame, 33 | * Drill Authentication using PAM, 34 | 35 | Installation 36 | ------------ 37 | 38 | Version from https://pypi.python.org/pypi/pydrill:: 39 | 40 | $ pip install pydrill 41 | 42 | Latest version from git:: 43 | 44 | $ pip install git+git://github.com/PythonicNinja/pydrill.git 45 | 46 | Sample usage 47 | ------------ 48 | :: 49 | 50 | from pydrill.client import PyDrill 51 | 52 | drill = PyDrill(host='localhost', port=8047) 53 | 54 | if not drill.is_active(): 55 | raise ImproperlyConfigured('Please run Drill first') 56 | 57 | yelp_reviews = drill.query(''' 58 | SELECT * FROM 59 | `dfs.root`.`./Users/macbookair/Downloads/yelp_dataset_challenge_academic_dataset/yelp_academic_dataset_review.json` 60 | LIMIT 5 61 | ''') 62 | 63 | for result in yelp_reviews: 64 | print("%s: %s" %(result['type'], result['date'])) 65 | 66 | 67 | # pandas dataframe 68 | 69 | df = yelp_reviews.to_dataframe() 70 | print(df[df['stars'] > 3]) 71 | -------------------------------------------------------------------------------- /docker_drill_embedded.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker_drill_embedded.sh 4 | 5 | docker run -it -p 8047:8047 mkieboom/apache-drill-docker /drill-scripts/bootstrap.sh 6 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pydrill.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pydrill.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pydrill" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pydrill" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # pydrill documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another 20 | # directory, add these directories to sys.path here. If the directory is 21 | # relative to the documentation root, use os.path.abspath to make it 22 | # absolute, like shown here. 23 | sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # Get the project root dir, which is the parent dir of this 26 | cwd = os.getcwd() 27 | project_root = os.path.dirname(cwd) 28 | 29 | # Insert the project root dir as the first element in the PYTHONPATH. 30 | # This lets us ensure that the source package is imported, and that its 31 | # version is used. 32 | sys.path.insert(0, project_root) 33 | 34 | import pydrill 35 | 36 | # -- General configuration --------------------------------------------- 37 | 38 | # If your documentation needs a minimal Sphinx version, state it here. 39 | #needs_sphinx = '1.0' 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 43 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # The suffix of source filenames. 49 | source_suffix = '.rst' 50 | 51 | # The encoding of source files. 52 | #source_encoding = 'utf-8-sig' 53 | 54 | # The master toctree document. 55 | master_doc = 'index' 56 | 57 | # General information about the project. 58 | project = u'pydrill' 59 | copyright = u'2015, Wojciech Nowak' 60 | 61 | # The version info for the project you're documenting, acts as replacement 62 | # for |version| and |release|, also used in various other places throughout 63 | # the built documents. 64 | # 65 | # The short X.Y version. 66 | version = pydrill.version 67 | # The full version, including alpha/beta/rc tags. 68 | release = pydrill.version 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | #language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to 75 | # some non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = ['_build'] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built 106 | # documents. 107 | #keep_warnings = False 108 | 109 | 110 | # -- Options for HTML output ------------------------------------------- 111 | 112 | # The theme to use for HTML and HTML Help pages. See the documentation for 113 | # a list of builtin themes. 114 | html_theme = 'furo' 115 | 116 | # Theme options are theme-specific and customize the look and feel of a 117 | # theme further. For a list of options available for each theme, see the 118 | # documentation. 119 | #html_theme_options = {} 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | #html_theme_path = [] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as 129 | # html_title. 130 | #html_short_title = None 131 | 132 | # The name of an image file (relative to this directory) to place at the 133 | # top of the sidebar. 134 | #html_logo = None 135 | 136 | # The name of an image file (within the static path) to use as favicon 137 | # of the docs. This file should be a Windows icon file (.ico) being 138 | # 16x16 or 32x32 pixels large. 139 | #html_favicon = None 140 | 141 | # Add any paths that contain custom static files (such as style sheets) 142 | # here, relative to this directory. They are copied after the builtin 143 | # static files, so a file named "default.css" will overwrite the builtin 144 | # "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page 148 | # bottom, using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names 159 | # to template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_domain_indices = True 164 | 165 | # If false, no index is generated. 166 | #html_use_index = True 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, "Created using Sphinx" is shown in the HTML footer. 175 | # Default is True. 176 | #html_show_sphinx = True 177 | 178 | # If true, "(C) Copyright ..." is shown in the HTML footer. 179 | # Default is True. 180 | #html_show_copyright = True 181 | 182 | # If true, an OpenSearch description file will be output, and all pages 183 | # will contain a tag referring to it. The value of this option 184 | # must be the base URL from which the finished HTML is served. 185 | #html_use_opensearch = '' 186 | 187 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 188 | #html_file_suffix = None 189 | 190 | # Output file base name for HTML help builder. 191 | htmlhelp_basename = 'pydrilldoc' 192 | 193 | 194 | # -- Options for LaTeX output ------------------------------------------ 195 | 196 | latex_elements = { 197 | # The paper size ('letterpaper' or 'a4paper'). 198 | #'papersize': 'letterpaper', 199 | 200 | # The font size ('10pt', '11pt' or '12pt'). 201 | #'pointsize': '10pt', 202 | 203 | # Additional stuff for the LaTeX preamble. 204 | #'preamble': '', 205 | } 206 | 207 | # Grouping the document tree into LaTeX files. List of tuples 208 | # (source start file, target name, title, author, documentclass 209 | # [howto/manual]). 210 | latex_documents = [ 211 | ('index', 'pydrill.tex', 212 | u'pydrill Documentation', 213 | u'Wojciech Nowak', 'manual'), 214 | ] 215 | 216 | # The name of an image file (relative to this directory) to place at 217 | # the top of the title page. 218 | #latex_logo = None 219 | 220 | # For "manual" documents, if this is true, then toplevel headings 221 | # are parts, not chapters. 222 | #latex_use_parts = False 223 | 224 | # If true, show page references after internal links. 225 | #latex_show_pagerefs = False 226 | 227 | # If true, show URL addresses after external links. 228 | #latex_show_urls = False 229 | 230 | # Documents to append as an appendix to all manuals. 231 | #latex_appendices = [] 232 | 233 | # If false, no module index is generated. 234 | #latex_domain_indices = True 235 | 236 | 237 | # -- Options for manual page output ------------------------------------ 238 | 239 | # One entry per manual page. List of tuples 240 | # (source start file, name, description, authors, manual section). 241 | man_pages = [ 242 | ('index', 'pydrill', 243 | u'pydrill Documentation', 244 | [u'Wojciech Nowak'], 1) 245 | ] 246 | 247 | # If true, show URL addresses after external links. 248 | #man_show_urls = False 249 | 250 | 251 | # -- Options for Texinfo output ---------------------------------------- 252 | 253 | # Grouping the document tree into Texinfo files. List of tuples 254 | # (source start file, target name, title, author, 255 | # dir menu entry, description, category) 256 | texinfo_documents = [ 257 | ('index', 'pydrill', 258 | u'pydrill Documentation', 259 | u'Wojciech Nowak', 260 | 'pydrill', 261 | 'One line description of project.', 262 | 'Miscellaneous'), 263 | ] 264 | 265 | # Documents to append as an appendix to all manuals. 266 | #texinfo_appendices = [] 267 | 268 | # If false, no module index is generated. 269 | #texinfo_domain_indices = True 270 | 271 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 272 | #texinfo_show_urls = 'footnote' 273 | 274 | # If true, do not generate a @detailmenu in the "Top" node's menu. 275 | #texinfo_no_detailmenu = False 276 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pydrill documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pydrill's documentation! 7 | ====================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | installation 16 | usage 17 | contributing 18 | authors 19 | history 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | 28 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Installation 5 | ============ 6 | 7 | At the command line:: 8 | 9 | $ easy_install pydrill 10 | 11 | Or, if you have virtualenvwrapper installed:: 12 | 13 | $ mkvirtualenv pydrill 14 | $ pip install pydrill 15 | 16 | Latest version from git:: 17 | 18 | $ pip install git+git://github.com/PythonicNinja/pydrill.git 19 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pydrill.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pydrill.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | Supported api calls 4 | ------------------- 5 | .. autoclass:: pydrill.client.PyDrill 6 | :members: 7 | :undoc-members: 8 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx==3.5.4 2 | furo==2021.4.11b34 3 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | Usage 3 | ===== 4 | 5 | To use pydrill in a project:: 6 | 7 | from pydrill.client import PyDrill 8 | 9 | drill = PyDrill(host='localhost', port=8047) 10 | 11 | You can also initialize via environment variables such as:: 12 | 13 | PYDRILL_HOST 14 | PYDRILL_PORT 15 | 16 | You can use Drill PAM authentication via auth param:: 17 | 18 | drill = PyDrill(auth='drill_user:drill_password') 19 | 20 | 21 | To enable specific storage plugin you can:: 22 | 23 | drill.storage_enable('mongo') 24 | 25 | You can view all queries which were executed or are running:: 26 | 27 | drill.profiles() 28 | 29 | 30 | To check if Drill is running:: 31 | 32 | if drill.is_active(): 33 | your_code 34 | 35 | 36 | Query involves only providing sql:: 37 | 38 | 39 | employees = drill.query(''' 40 | SELECT * FROM cp.`employee.json` LIMIT 5 41 | ''') 42 | 43 | for employee in employees: 44 | print result 45 | 46 | If you feel like building sql queries is not nicest thing ever you should try pydrill_dsl https://pypi.python.org/pypi/pydrill_dsl 47 | 48 | Support for pandas:: 49 | 50 | # pandas dataframe 51 | df = employees.to_dataframe() 52 | print(df[df['salary'] > 20000]) 53 | 54 | Supported api calls 55 | ------------------- 56 | .. autoclass:: pydrill.client.PyDrill 57 | :members: 58 | :undoc-members: 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /pydrill/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | version = '0.3.4' 4 | 5 | VERSION = tuple(map(int, version.split('.'))) 6 | __version__ = VERSION 7 | __versionstr__ = version 8 | 9 | 10 | if (2, 7) <= sys.version_info < (3, 6): 11 | # 12 | import logging 13 | logger = logging.getLogger('pydrill') 14 | logger.addHandler(logging.NullHandler()) 15 | -------------------------------------------------------------------------------- /pydrill/client/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | from pydrill.transport import Transport 5 | from pydrill.client.result import ResultQuery, Result, Stats, Profiles 6 | from pydrill.connection.requests_conn import RequestsHttpConnection 7 | from pydrill.exceptions import QueryError, ConnectionError, TransportError 8 | 9 | 10 | class PyDrill(object): 11 | """ 12 | >>> drill = PyDrill(host='localhost', port=8047) 13 | >>> drill.is_active() 14 | True 15 | """ 16 | # TODO: create better docs. 17 | 18 | def __init__(self, host=os.environ.get('PYDRILL_HOST', 'localhost'), port=os.environ.get('PYDRILL_PORT', 8047), 19 | trasport_class=Transport, connection_class=RequestsHttpConnection, auth=None, **kwargs): 20 | 21 | self.transport = trasport_class(host, port, connection_class=connection_class, auth=auth, **kwargs) 22 | 23 | def perform_request(self, method, url, params=None, body=None): 24 | return self.transport.perform_request(method, url, params, body) 25 | 26 | def is_active(self, timeout=2): 27 | """ 28 | :param timeout: int 29 | :return: boolean 30 | """ 31 | try: 32 | result = Result(*self.perform_request('HEAD', '/', params={'request_timeout': timeout})) 33 | except ConnectionError: 34 | return False 35 | except TransportError: 36 | return False 37 | 38 | if result.response.status_code == 200: 39 | return True 40 | 41 | return False 42 | 43 | def query(self, sql, timeout=10): 44 | """ 45 | Submit a query and return results. 46 | 47 | :param sql: string 48 | :param timeout: int 49 | :return: pydrill.client.ResultQuery 50 | """ 51 | if not sql: 52 | raise QueryError('No query passed to drill.') 53 | 54 | result = ResultQuery(*self.perform_request(**{ 55 | 'method': 'POST', 56 | 'url': '/query.json', 57 | 'body': { 58 | "queryType": "SQL", 59 | "query": sql 60 | }, 61 | 'params': { 62 | 'request_timeout': timeout 63 | } 64 | })) 65 | 66 | return result 67 | 68 | def plan(self, sql, timeout=10): 69 | """ 70 | :param sql: string 71 | :param timeout: int 72 | :return: pydrill.client.ResultQuery 73 | """ 74 | sql = 'explain plan for ' + sql 75 | return self.query(sql, timeout) 76 | 77 | def stats(self, timeout=10): 78 | """ 79 | Get Drillbit information, such as ports numbers. 80 | 81 | :param timeout: int 82 | :return: pydrill.client.Stats 83 | """ 84 | result = Stats(*self.perform_request(**{ 85 | 'method': 'GET', 86 | 'url': '/stats.json', 87 | 'params': { 88 | 'request_timeout': timeout 89 | } 90 | })) 91 | return result 92 | 93 | def metrics(self, timeout=10): 94 | """ 95 | Get the current memory metrics. 96 | 97 | :param timeout: int 98 | :return: pydrill.client.Result 99 | """ 100 | result = Result(*self.perform_request(**{ 101 | 'method': 'GET', 102 | 'url': '/status/metrics', 103 | 'params': { 104 | 'request_timeout': timeout 105 | } 106 | })) 107 | return result 108 | 109 | def threads(self, timeout=10): 110 | """ 111 | Get the status of threads. 112 | 113 | :param timeout: int 114 | :return: pydrill.client.Result 115 | """ 116 | result = Result(*self.perform_request(**{ 117 | 'method': 'GET', 118 | 'url': '/status/threads', 119 | 'params': { 120 | 'request_timeout': timeout 121 | } 122 | })) 123 | return result 124 | 125 | def options(self, timeout=10): 126 | """ 127 | List the name, default, and data type of the system and session options. 128 | 129 | :param timeout: int 130 | :return: pydrill.client.Result 131 | """ 132 | result = Result(*self.perform_request(**{ 133 | 'method': 'GET', 134 | 'url': '/options.json', 135 | 'params': { 136 | 'request_timeout': timeout 137 | } 138 | })) 139 | return result 140 | 141 | def storage(self, timeout=10): 142 | """ 143 | Get the list of storage plugin names and configurations. 144 | 145 | :param timeout: int 146 | :return: pydrill.client.Result 147 | """ 148 | result = Result(*self.perform_request(**{ 149 | 'method': 'GET', 150 | 'url': '/storage.json', 151 | 'params': { 152 | 'request_timeout': timeout 153 | } 154 | })) 155 | return result 156 | 157 | def storage_detail(self, name, timeout=10): 158 | """ 159 | Get the definition of the named storage plugin. 160 | 161 | :param name: The assigned name in the storage plugin definition. 162 | :param timeout: int 163 | :return: pydrill.client.Result 164 | """ 165 | result = Result(*self.perform_request(**{ 166 | 'method': 'GET', 167 | 'url': '/storage/{0}.json'.format(name), 168 | 'params': { 169 | 'request_timeout': timeout 170 | } 171 | })) 172 | return result 173 | 174 | def storage_enable(self, name, value=True, timeout=10): 175 | """ 176 | Enable or disable the named storage plugin. 177 | 178 | :param name: The assigned name in the storage plugin definition. 179 | :param value: Either True (to enable) or False (to disable). 180 | :param timeout: int 181 | :return: pydrill.client.Result 182 | """ 183 | value = 'true' if value else 'false' 184 | result = Result(*self.perform_request(**{ 185 | 'method': 'GET', 186 | 'url': '/storage/{0}/enable/{1}'.format(name, value), 187 | 'params': { 188 | 'request_timeout': timeout 189 | } 190 | })) 191 | return result 192 | 193 | def storage_update(self, name, config, timeout=10): 194 | """ 195 | Create or update a storage plugin configuration. 196 | 197 | :param name: The name of the storage plugin configuration to create or update. 198 | :param config: Overwrites the existing configuration if there is any, and therefore, must include all 199 | required attributes and definitions. 200 | :param timeout: int 201 | :return: pydrill.client.Result 202 | """ 203 | result = Result(*self.perform_request(**{ 204 | 'method': 'POST', 205 | 'url': '/storage/{0}.json'.format(name), 206 | 'body': config, 207 | 'params': { 208 | 'request_timeout': timeout 209 | } 210 | })) 211 | return result 212 | 213 | def storage_delete(self, name, timeout=10): 214 | """ 215 | Delete a storage plugin configuration. 216 | 217 | :param name: The name of the storage plugin configuration to delete. 218 | :param timeout: int 219 | :return: pydrill.client.Result 220 | """ 221 | result = Result(*self.perform_request(**{ 222 | 'method': 'DELETE', 223 | 'url': '/storage/{0}.json'.format(name), 224 | 'params': { 225 | 'request_timeout': timeout 226 | } 227 | })) 228 | return result 229 | 230 | def profiles(self, timeout=10): 231 | """ 232 | Get the profiles of running and completed queries. 233 | 234 | :param timeout: int 235 | :return: pydrill.client.Result 236 | """ 237 | result = Profiles(*self.perform_request(**{ 238 | 'method': 'GET', 239 | 'url': '/profiles.json', 240 | 'params': { 241 | 'request_timeout': timeout 242 | } 243 | })) 244 | return result 245 | 246 | def profile(self, query_id, timeout=10): 247 | """ 248 | Get the profile of the query that has the given queryid. 249 | 250 | :param query_id: The UUID of the query in standard UUID format that Drill assigns to each query. 251 | :param timeout: int 252 | :return: pydrill.client.Result 253 | """ 254 | result = Result(*self.perform_request(**{ 255 | 'method': 'GET', 256 | 'url': '/profiles/{0}.json'.format(query_id), 257 | 'params': { 258 | 'request_timeout': timeout 259 | } 260 | })) 261 | return result 262 | 263 | def profile_cancel(self, query_id, timeout=10): 264 | """ 265 | Cancel the query that has the given queryid. 266 | 267 | :param query_id: The UUID of the query in standard UUID format that Drill assigns to each query. 268 | :param timeout: int 269 | :return: pydrill.client.Result 270 | """ 271 | result = Result(*self.perform_request(**{ 272 | 'method': 'GET', 273 | 'url': '/profiles/cancel/{0}'.format(query_id), 274 | 'params': { 275 | 'request_timeout': timeout 276 | } 277 | })) 278 | return result 279 | -------------------------------------------------------------------------------- /pydrill/client/result.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from pydrill.exceptions import ImproperlyConfigured 3 | 4 | try: 5 | import pandas as pd 6 | 7 | PANDAS_AVAILABLE = True 8 | except ImportError: 9 | PANDAS_AVAILABLE = False 10 | 11 | 12 | class Result(object): 13 | def __init__(self, response, data, duration, *args, **kwargs): 14 | self.response = response 15 | self.duration = duration 16 | self.data = data 17 | 18 | 19 | class ResultQuery(Result): 20 | """ 21 | Class responsible for maintaining a information returned from Drill. 22 | 23 | It is iterable. 24 | """ 25 | def __init__(self, response, data, duration, *args, **kwargs): 26 | super(ResultQuery, self).__init__(response, data, duration, *args, **kwargs) 27 | self.rows = data.get('rows', []) 28 | self.columns = data.get('columns', []) 29 | 30 | def __iter__(self): 31 | for row in self.rows: 32 | yield row 33 | 34 | def to_dataframe(self, dtype=None): 35 | if not PANDAS_AVAILABLE: 36 | raise ImproperlyConfigured("Please install pandas to use ResultQuery.to_dataframe().") 37 | return pd.DataFrame.from_dict(self.rows, dtype=dtype) 38 | 39 | 40 | class Drillbit(object): 41 | def __init__(self, id, address, status, *args, **kwargs): 42 | self.id = id 43 | self.address = address 44 | self.status = status 45 | 46 | 47 | class Stats(Result): 48 | def __init__(self, response, data, duration, *args, **kwargs): 49 | super(Stats, self).__init__(response, data, duration, *args, **kwargs) 50 | 51 | self.drillbits = [] 52 | 53 | for metric in data: 54 | value, name = metric['value'], metric['name'] 55 | 56 | if name == 'Number of Drill Bits': 57 | self.drillbits_number = value 58 | elif name.startswith('Bit #'): 59 | address, status = value.split() 60 | self.drillbits.append(Drillbit(id=name.split('#')[-1], address=address, status=status)) 61 | elif name == 'Data Port Address': 62 | self.data_port_address = value 63 | elif name == 'User Port Address': 64 | self.user_port_address = value 65 | elif name == 'Control Port Address': 66 | self.control_port_address = value 67 | elif name == 'Maximum Direct Memory': 68 | self.max_direct_memory = value 69 | 70 | 71 | class Profiles(Result): 72 | def __init__(self, response, data, duration, *args, **kwargs): 73 | super(Profiles, self).__init__(response, data, duration, *args, **kwargs) 74 | self.running_queries = data.get('runningQueries') 75 | self.finished_queries = data.get('finishedQueries') 76 | -------------------------------------------------------------------------------- /pydrill/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | 5 | PY2 = sys.version_info[0] == 2 6 | 7 | if PY2: 8 | string_types = basestring, # noqa 9 | from urllib import quote_plus, urlencode # noqa 10 | from urlparse import urlparse # noqa 11 | from itertools import imap as map # noqa 12 | else: 13 | string_types = str, bytes # noqa 14 | from urllib.parse import quote_plus, urlencode, urlparse # noqa 15 | map = map # noqa 16 | -------------------------------------------------------------------------------- /pydrill/connection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PythonicNinja/pydrill/5dd6b7861937f7c3da9c594e792a31f57f5de617/pydrill/connection/__init__.py -------------------------------------------------------------------------------- /pydrill/connection/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | 5 | from ..exceptions import HTTP_EXCEPTIONS, TransportError 6 | 7 | try: 8 | import simplejson as json 9 | except ImportError: 10 | import json 11 | 12 | 13 | logger = logging.getLogger('pydrill') 14 | 15 | _tracer_already_configured = 'pydrill.trace' in logging.Logger.manager.loggerDict 16 | tracer = logging.getLogger('pydrill.trace') 17 | if not _tracer_already_configured: 18 | tracer.propagate = False 19 | 20 | 21 | class Connection(object): 22 | """ 23 | Class responsible for maintaining a connection to an Drill node. 24 | You can create custom connection class similar to :class:`~pydrill.RequestsHttpConnection` 25 | 26 | 27 | It's main interface (`perform_request`) is thread-safe. 28 | Responsible for logging. 29 | """ 30 | transport_schema = 'http' 31 | 32 | def __init__(self, host='localhost', port=8047, url_prefix='', timeout=10, **kwargs): 33 | """ 34 | :arg host: hostname of the node (default: localhost) 35 | :arg port: port to use (integer, default: 8047) 36 | :arg url_prefix: optional url prefix for pydrill 37 | :arg timeout: default timeout in seconds (float, default: 10) 38 | """ 39 | self.host = '%s://%s:%s' % (self.transport_schema, host, port) 40 | if url_prefix: 41 | url_prefix = '/' + url_prefix.strip('/') 42 | self.url_prefix = url_prefix 43 | self.timeout = timeout 44 | 45 | def __repr__(self): 46 | return '<%s: %s>' % (self.__class__.__name__, self.host) 47 | 48 | def _pretty_json(self, data): 49 | """ 50 | pretty JSON in tracer curl logs 51 | 52 | :param data: 53 | :return: 54 | """ 55 | try: 56 | return json.dumps(json.loads(data), sort_keys=True, indent=2, separators=(',', ': ')).replace("'", 57 | r'\u0027') 58 | except (ValueError, TypeError): 59 | # non-json data or a bulk request 60 | return data 61 | 62 | def log_request_success(self, method, full_url, path, body, status_code, response, duration): 63 | """ Log a successful API call. """ 64 | 65 | if body and not isinstance(body, dict): 66 | body = body.decode('utf-8') 67 | 68 | logger.info( 69 | '%s %s [status:%s request:%.3fs]', method, full_url, 70 | status_code, duration 71 | ) 72 | logger.debug('> %s', body) 73 | logger.debug('< %s', response) 74 | 75 | if tracer.isEnabledFor(logging.INFO): 76 | if self.url_prefix: 77 | path = path.replace(self.url_prefix, '', 1) 78 | tracer.info("curl -X%s 'http://localhost:8047%s' -d '%s'", method, path, 79 | self._pretty_json(body) if body else '') 80 | 81 | if tracer.isEnabledFor(logging.DEBUG): 82 | tracer.debug('#[%s] (%.3fs)\n#%s', status_code, duration, 83 | self._pretty_json(response).replace('\n', '\n#') if response else '') 84 | 85 | def log_request_fail(self, method, full_url, body, duration, status_code=None, exception=None): 86 | """ 87 | Log an unsuccessful API call. 88 | """ 89 | logger.warning( 90 | '%s %s [status:%s request:%.3fs]', method, full_url, 91 | status_code or 'N/A', duration, exc_info=exception is not None 92 | ) 93 | 94 | if body and not isinstance(body, dict): 95 | body = body.decode('utf-8') 96 | 97 | logger.debug('> %s', body) 98 | 99 | def _raise_error(self, status_code, raw_data): 100 | """ 101 | Locate appropriate exception and raise it. 102 | """ 103 | error_message = raw_data 104 | additional_info = None 105 | try: 106 | additional_info = json.loads(raw_data) 107 | error_message = additional_info.get('error', error_message) 108 | if isinstance(error_message, dict) and 'type' in error_message: 109 | error_message = error_message['type'] 110 | except: 111 | pass 112 | 113 | raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info) 114 | 115 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()): 116 | raise NotImplementedError 117 | -------------------------------------------------------------------------------- /pydrill/connection/requests_conn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | import warnings 5 | 6 | from ..compat import string_types, urlencode 7 | from ..exceptions import ConnectionError, ConnectionTimeout, ImproperlyConfigured, SSLError, SerializationError 8 | from .base import Connection 9 | 10 | try: 11 | import requests 12 | 13 | REQUESTS_AVAILABLE = True 14 | except ImportError: 15 | REQUESTS_AVAILABLE = False 16 | 17 | 18 | class RequestsHttpConnection(Connection): 19 | """ 20 | Connection using the `requests` library. 21 | :arg http_auth: optional http auth information as either ':' separated 22 | string or a tuple. Any value will be passed into requests as `auth`. 23 | :arg use_ssl: use ssl for the connection if `True` 24 | :arg verify_certs: whether to verify SSL certificates 25 | :arg ca_certs: optional path to CA bundle. By default standard requests' 26 | bundle will be used. 27 | :arg client_cert: path to the file containing the private key and the 28 | certificate 29 | """ 30 | 31 | def __init__(self, host='localhost', port=8047, auth=None, 32 | use_ssl=False, verify_certs=False, ca_certs=None, client_cert=None, 33 | **kwargs): 34 | if not REQUESTS_AVAILABLE: 35 | raise ImproperlyConfigured("Please install requests to use RequestsHttpConnection.") 36 | 37 | super(RequestsHttpConnection, self).__init__(host=host, port=port, **kwargs) 38 | self.session = requests.session() 39 | if auth is not None: 40 | if isinstance(auth, (tuple, list)): 41 | auth = tuple(auth) 42 | elif isinstance(auth, string_types): 43 | auth = tuple(auth.split(':', 1)) 44 | self.session.auth = auth 45 | self.base_url = 'http%s://%s:%s%s' % ( 46 | 's' if use_ssl else '', 47 | host, port, self.url_prefix 48 | ) 49 | self.session.verify = verify_certs 50 | self.session.cert = client_cert 51 | if ca_certs: 52 | if not verify_certs: 53 | raise ImproperlyConfigured("You cannot pass CA certificates when verify SSL is off.") 54 | self.session.verify = ca_certs 55 | 56 | if use_ssl and not verify_certs: 57 | warnings.warn('Connecting to %s using SSL with verify_certs=False is insecure.' % self.base_url) 58 | 59 | if auth is not None: 60 | try: 61 | self.perform_request( 62 | method='POST', 63 | url='/j_security_check', 64 | body={'j_username': auth[0], 'j_password': auth[1]}, 65 | headers={} 66 | ) 67 | except SerializationError: 68 | warnings.warn('Authentication failed using %s', auth) 69 | 70 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None): 71 | url = self.base_url + url 72 | if params: 73 | url = '%s?%s' % (url, urlencode(params or {})) 74 | 75 | if timeout is None: 76 | timeout = self.timeout 77 | 78 | if headers is None: 79 | headers = {'Content-Type': 'application/json'} 80 | 81 | start = time.time() 82 | try: 83 | response = self.session.request( 84 | method, url, 85 | data=body, 86 | headers=headers, 87 | timeout=timeout, 88 | ) 89 | response.encoding = 'UTF-8' 90 | duration = time.time() - start 91 | raw_data = response.text 92 | except requests.exceptions.SSLError as e: 93 | self.log_request_fail(method, url, body, time.time() - start, exception=e) 94 | raise SSLError('N/A', str(e), e) 95 | except requests.Timeout as e: 96 | self.log_request_fail(method, url, body, time.time() - start, exception=e) 97 | raise ConnectionTimeout('TIMEOUT', str(e), e) 98 | except requests.ConnectionError as e: 99 | self.log_request_fail(method, url, body, time.time() - start, exception=e) 100 | raise ConnectionError('N/A', str(e), e) 101 | 102 | # raise errors based on http status codes, let the client handle those if needed 103 | if not (200 <= response.status_code < 300) and response.status_code not in ignore: 104 | self.log_request_fail(method, url, body, duration, response.status_code) 105 | self._raise_error(response.status_code, raw_data) 106 | 107 | if 'action="/j_security_check"' in raw_data: 108 | self.log_request_fail(method, url, body, duration, response.status_code) 109 | error_message = raw_data.split('

')[1].split('

')[0] 110 | self._raise_error(response.status_code, error_message) 111 | 112 | self.log_request_success(method, url, response.request.path_url, body, response.status_code, raw_data, duration) 113 | 114 | return response, raw_data, duration 115 | -------------------------------------------------------------------------------- /pydrill/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __all__ = [ 4 | 'ImproperlyConfigured', 'PyDrillException', 'SerializationError', 5 | 'TransportError', 'NotFoundError', 'ConflictError', 'RequestError', 'ConnectionError', 6 | 'SSLError', 'ConnectionTimeout' 7 | ] 8 | 9 | 10 | class ImproperlyConfigured(Exception): 11 | """ 12 | Exception raised when the config passed to the client is inconsistent or invalid. 13 | """ 14 | 15 | 16 | class PyDrillException(Exception): 17 | """ 18 | Base class for all exceptions raised by this package's operations (doesn't 19 | apply to :class:`~pydrill.ImproperlyConfigured`). 20 | """ 21 | 22 | 23 | class SerializationError(PyDrillException): 24 | """ 25 | Data passed in failed to serialize properly in the ``Serializer`` being 26 | used. 27 | """ 28 | 29 | 30 | class TransportError(PyDrillException): 31 | """ 32 | Exception raised when Drill returns a non-OK (>=400) HTTP status code. Or when 33 | an actual connection error happens; in that case the ``status_code`` will 34 | be set to ``'N/A'``. 35 | """ 36 | @property 37 | def status_code(self): 38 | """ 39 | The HTTP status code of the response that precipitated the error or 40 | ``'N/A'`` if not applicable. 41 | """ 42 | return self.args[0] 43 | 44 | @property 45 | def error(self): 46 | """ A string error message. """ 47 | return self.args[1] 48 | 49 | @property 50 | def info(self): 51 | """ Dict of returned error info from Drill, where available. """ 52 | return self.args[2] 53 | 54 | def __str__(self): 55 | cause = '' 56 | try: 57 | if self.info: 58 | cause = ', %r' % self.info['error']['root_cause'][0]['reason'] 59 | except LookupError: 60 | pass 61 | return 'TransportError(%s, %r%s)' % (self.status_code, self.error, cause) 62 | 63 | 64 | class QueryError(PyDrillException): 65 | """ 66 | Error raised when there was an exception in query. 67 | """ 68 | 69 | 70 | class ConnectionError(TransportError): 71 | """ 72 | Error raised when there was an exception while talking to Drill. Original 73 | exception from the underlying :class:`~pydrill.Connection` 74 | implementation is available as ``.info.`` 75 | """ 76 | def __str__(self): 77 | return 'ConnectionError(%s) caused by: %s(%s)' % ( 78 | self.error, self.info.__class__.__name__, self.info) 79 | 80 | 81 | class SSLError(ConnectionError): 82 | """ Error raised when encountering SSL errors. """ 83 | 84 | 85 | class ConnectionTimeout(ConnectionError): 86 | """ A network timeout. Doesn't cause a node retry by default. """ 87 | def __str__(self): 88 | return 'ConnectionTimeout caused by - %s(%s)' % ( 89 | self.info.__class__.__name__, self.info) 90 | 91 | 92 | class NotFoundError(TransportError): 93 | """ Exception representing a 404 status code. """ 94 | 95 | 96 | class ConflictError(TransportError): 97 | """ Exception representing a 409 status code. """ 98 | 99 | 100 | class RequestError(TransportError): 101 | """ Exception representing a 400 status code. """ 102 | 103 | 104 | class AuthenticationException(TransportError): 105 | """ Exception representing a 401 status code. """ 106 | 107 | 108 | class AuthorizationException(TransportError): 109 | """ Exception representing a 403 status code. """ 110 | 111 | """ 112 | Mapping used by base class :class:`~pydrill.Connection` with method `_raise_error` 113 | """ 114 | HTTP_EXCEPTIONS = { 115 | 400: RequestError, 116 | 401: AuthenticationException, 117 | 403: AuthorizationException, 118 | 404: NotFoundError, 119 | 409: ConflictError, 120 | } 121 | -------------------------------------------------------------------------------- /pydrill/serializer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | try: 4 | import simplejson as json 5 | except ImportError: 6 | import json 7 | 8 | import uuid 9 | from datetime import date, datetime 10 | from decimal import Decimal 11 | 12 | from .compat import string_types 13 | from .exceptions import ImproperlyConfigured, SerializationError 14 | 15 | 16 | class TextSerializer(object): 17 | mimetype = 'text/plain' 18 | 19 | def loads(self, s): 20 | return s 21 | 22 | def dumps(self, data): 23 | if isinstance(data, string_types): 24 | return data 25 | 26 | raise SerializationError('Cannot serialize %r into text.' % data) 27 | 28 | 29 | class JSONSerializer(object): 30 | mimetype = 'application/json' 31 | 32 | def default(self, data): 33 | if isinstance(data, (date, datetime)): 34 | return data.isoformat() 35 | elif isinstance(data, Decimal): 36 | return float(data) 37 | elif isinstance(data, uuid.UUID): 38 | return str(data) 39 | raise TypeError("Unable to serialize %r (type: %s)" % (data, type(data))) 40 | 41 | def loads(self, s): 42 | try: 43 | return json.loads(s) 44 | except (ValueError, TypeError) as e: 45 | raise SerializationError(s, e) 46 | 47 | def dumps(self, data): 48 | # don't serialize strings 49 | if isinstance(data, string_types): 50 | return data 51 | 52 | try: 53 | return json.dumps(data, default=self.default, ensure_ascii=False) 54 | except (ValueError, TypeError) as e: 55 | raise SerializationError(data, e) 56 | 57 | 58 | DEFAULT_SERIALIZERS = { 59 | JSONSerializer.mimetype: JSONSerializer(), 60 | TextSerializer.mimetype: TextSerializer(), 61 | } 62 | 63 | 64 | class Deserializer(object): 65 | def __init__(self, serializers=DEFAULT_SERIALIZERS, default_mimetype='application/json'): 66 | try: 67 | self.default = serializers[default_mimetype] 68 | except KeyError: 69 | raise ImproperlyConfigured('Cannot find default serializer (%s)' % default_mimetype) 70 | self.serializers = serializers 71 | 72 | def loads(self, s, mimetype=None): 73 | if not mimetype: 74 | deserializer = self.default 75 | else: 76 | # split out charset 77 | mimetype = mimetype.split(';', 1)[0] 78 | try: 79 | deserializer = self.serializers[mimetype] 80 | except KeyError: 81 | raise SerializationError('Unknown mimetype, unable to deserialize: %s' % mimetype) 82 | 83 | return deserializer.loads(s) 84 | -------------------------------------------------------------------------------- /pydrill/transport.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | from pydrill.exceptions import ConnectionError, ConnectionTimeout, TransportError 5 | from pydrill.serializer import Deserializer, JSONSerializer 6 | 7 | 8 | class Transport(object): 9 | """ 10 | Encapsulation of transport-related to logic. Handles instantiation of the 11 | individual connection. 12 | Main interface is the `perform_request` method. 13 | """ 14 | 15 | def __init__(self, host, port, connection_class, serializers=None, default_mimetype='application/json', 16 | max_retries=3, retry_on_status=(503, 504,), serializer=JSONSerializer(), deserializer=Deserializer(), 17 | retry_on_timeout=False, send_get_body_as='GET', **kwargs): 18 | self.deserializer = deserializer 19 | self.port = port 20 | self.host = host 21 | self.connection = connection_class(host, port, **kwargs) 22 | self.retry_on_status = retry_on_status 23 | self.serializers = serializers 24 | self.retry_on_timeout = retry_on_timeout 25 | self.default_mimetype = default_mimetype 26 | self.max_retries = max_retries 27 | self.send_get_body_as = send_get_body_as 28 | self.serializer = serializer 29 | 30 | def perform_request(self, method, url, params=None, body=None): 31 | """ 32 | Perform the actual request. 33 | Retrieve a connection. 34 | Pass all the information to it's perform_request method and return the data. 35 | 36 | :arg method: HTTP method to use 37 | :arg url: absolute url (without host) to target 38 | :arg params: dictionary of query parameters, will be handed over to the 39 | underlying :class:`~pydrill.Connection` class for serialization 40 | :arg body: body of the request, will be serializes using serializer and 41 | passed to the connection 42 | """ 43 | if body is not None: 44 | body = self.serializer.dumps(body) 45 | 46 | # some clients or environments don't support sending GET with body 47 | if method in ('HEAD', 'GET') and self.send_get_body_as != 'GET': 48 | # send it as post instead 49 | if self.send_get_body_as == 'POST': 50 | method = 'POST' 51 | 52 | # or as source parameter 53 | elif self.send_get_body_as == 'source': 54 | if params is None: 55 | params = {} 56 | params['source'] = body 57 | body = None 58 | 59 | if body is not None: 60 | try: 61 | body = body.encode('utf-8') 62 | except (UnicodeDecodeError, AttributeError): 63 | # bytes/str - no need to re-encode 64 | pass 65 | 66 | ignore = () 67 | timeout = None 68 | if params: 69 | timeout = params.pop('request_timeout', None) 70 | ignore = params.pop('ignore', ()) 71 | if isinstance(ignore, int): 72 | ignore = (ignore,) 73 | 74 | for attempt in range(self.max_retries + 1): 75 | connection = self.get_connection() 76 | 77 | try: 78 | response, data, duration = connection.perform_request(method, url, params, body, ignore=ignore, 79 | timeout=timeout) 80 | except TransportError as e: 81 | retry = False 82 | if isinstance(e, ConnectionTimeout): 83 | retry = self.retry_on_timeout 84 | elif isinstance(e, ConnectionError): 85 | retry = True 86 | elif e.status_code in self.retry_on_status: 87 | retry = True 88 | 89 | if retry: 90 | if attempt == self.max_retries: 91 | raise 92 | else: 93 | raise 94 | else: 95 | if data: 96 | data = self.deserializer.loads(data, mimetype=response.headers.get('Content-Type')) 97 | else: 98 | data = {} 99 | return response, data, duration 100 | 101 | def get_connection(self): 102 | return self.connection 103 | -------------------------------------------------------------------------------- /requirements_base.txt: -------------------------------------------------------------------------------- 1 | requests 2 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements_base.txt 2 | bumpversion==0.5.3 3 | wheel==0.23.0 4 | watchdog==0.8.3 5 | cryptography==3.3.2 6 | PyYAML==5.4 7 | -r requirements_testing.txt 8 | -r docs/requirements.txt 9 | -------------------------------------------------------------------------------- /requirements_testing.txt: -------------------------------------------------------------------------------- 1 | requests 2 | responses 3 | pytest 4 | pytest-cov 5 | pytest-sugar 6 | flake8==2.4.1 7 | tox==2.1.1 8 | coverage==5.5 9 | -------------------------------------------------------------------------------- /run_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | exists() 4 | { 5 | command -v "$1" >/dev/null 2>&1 6 | } 7 | 8 | if exists docker-machine; then 9 | if [ "$(docker-machine status)" != "Running" ]; then 10 | docker-machine start 11 | fi 12 | eval "$(docker-machine env default)" 13 | fi 14 | 15 | screen -dm bash -c "./docker_drill_embedded.sh" 16 | 17 | CID=$(docker ps | grep apache-drill | cut -d ' ' -f 1) 18 | while [ "$CID" == "" ]; 19 | do 20 | CID=$(docker ps | grep apache-drill | cut -d ' ' -f 1) 21 | echo $(docker ps) 22 | sleep 0.25 23 | done 24 | 25 | echo 'Docker cid' $CID 26 | 27 | if exists docker-machine; then 28 | PYDRILL_HOST=$(docker-machine ip) 29 | PYDRILL_PORT=8047 30 | else 31 | PYDRILL_HOST='localhost' 32 | PYDRILL_PORT=8047 33 | fi 34 | 35 | export CID=$CID 36 | export PYDRILL_HOST=$PYDRILL_HOST 37 | export PYDRILL_PORT=$PYDRILL_PORT 38 | 39 | echo "PYDRILL_HOST:" $PYDRILL_HOST 40 | echo "PYDRILL_PORT:" $PYDRILL_PORT 41 | 42 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | 4 | [isort] 5 | multi_line_output = 5 6 | line_length=120 7 | known_first_party=project,store 8 | default_section=THIRDPARTY 9 | skip=.tox,.ve,dist,build 10 | 11 | [flake8] 12 | exclude = .ve,.tox,.git,docs,dist,build 13 | ignore = E123,E128,E731 14 | max-line-length = 120 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import find_packages 5 | try: 6 | from setuptools import setup 7 | except ImportError: 8 | from distutils.core import setup 9 | 10 | 11 | with open('README.rst') as readme_file: 12 | readme = readme_file.read() 13 | 14 | with open('HISTORY.rst') as history_file: 15 | history = history_file.read() 16 | 17 | requirements = open('requirements_base.txt', 'r').readlines() 18 | 19 | test_requirements = open('requirements_testing.txt', 'r').readlines() 20 | 21 | setup( 22 | name='pydrill', 23 | version='0.3.4', 24 | description="Python Driver for Apache Drill.", 25 | long_description=readme + '\n\n' + history, 26 | author="Wojciech Nowak", 27 | author_email='mail@pythonic.ninja', 28 | url='https://github.com/PythonicNinja/pydrill', 29 | packages=find_packages( 30 | where='.', 31 | exclude=('test_*', ) 32 | ), 33 | include_package_data=True, 34 | install_requires=requirements, 35 | license="MIT", 36 | zip_safe=False, 37 | keywords='pydrill', 38 | classifiers=[ 39 | 'Development Status :: 2 - Pre-Alpha', 40 | 'Intended Audience :: Developers', 41 | 'License :: OSI Approved :: MIT License', 42 | 'Natural Language :: English', 43 | "Programming Language :: Python :: 2", 44 | 'Programming Language :: Python :: 2.6', 45 | 'Programming Language :: Python :: 2.7', 46 | 'Programming Language :: Python :: 3', 47 | 'Programming Language :: Python :: 3.3', 48 | 'Programming Language :: Python :: 3.4', 49 | 'Programming Language :: Python :: 3.5', 50 | 'Programming Language :: Python :: 3.6', 51 | ], 52 | test_suite='tests', 53 | tests_require=test_requirements 54 | ) 55 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydrill.client import PyDrill 3 | 4 | 5 | @pytest.fixture(scope='function', autouse=True) 6 | def pydrill_instance(): 7 | drill = PyDrill() 8 | return drill 9 | 10 | 11 | @pytest.fixture() 12 | def pydrill_url(pydrill_instance): 13 | """ 14 | :type pydrill_instance: pydrill.client.PyDrill 15 | """ 16 | return pydrill_instance.transport.connection.base_url 17 | -------------------------------------------------------------------------------- /tests/test_authentication.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import pytest 4 | import responses 5 | 6 | from pydrill.client import PyDrill 7 | from pydrill.exceptions import TransportError 8 | 9 | 10 | @responses.activate 11 | def test_authentication_success(pydrill_url): 12 | 13 | responses.add(**{ 14 | 'method': responses.POST, 15 | 'url': "{0}/{1}".format(pydrill_url, 'j_security_check'), 16 | }) 17 | 18 | PyDrill(auth='user:password') 19 | 20 | 21 | @responses.activate 22 | def test_authentication_failure(): 23 | with pytest.raises(TransportError): 24 | PyDrill(auth='user:password') 25 | -------------------------------------------------------------------------------- /tests/test_metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from pydrill.client.result import Stats, Result, Drillbit 4 | 5 | 6 | def test_stats(pydrill_instance): 7 | """ 8 | :type pydrill_instance: pydrill.client.PyDrill 9 | """ 10 | stats = pydrill_instance.stats() 11 | assert type(stats) == Stats 12 | assert stats.response.status_code == 200 13 | assert stats.max_direct_memory 14 | 15 | assert stats.control_port_address 16 | assert stats.user_port_address 17 | assert stats.data_port_address 18 | 19 | assert stats.drillbits_number 20 | assert type(stats.drillbits) == type([]) 21 | assert type(stats.drillbits[0]) == Drillbit 22 | 23 | 24 | def test_metrics(pydrill_instance): 25 | """ 26 | :type pydrill_instance: pydrill.client.PyDrill 27 | """ 28 | result = pydrill_instance.metrics() 29 | assert type(result) == Result 30 | assert result.response.status_code == 200 31 | assert 'gauges' in result.data.keys() 32 | -------------------------------------------------------------------------------- /tests/test_options.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pydrill.client.result import Result 5 | 6 | 7 | def test_options(pydrill_instance): 8 | """ 9 | :type pydrill_instance: pydrill.client.PyDrill 10 | """ 11 | threads = pydrill_instance.options() 12 | assert type(threads) == Result 13 | assert threads.response.status_code == 200 14 | -------------------------------------------------------------------------------- /tests/test_profiles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import responses 4 | import uuid 5 | from pydrill.client.result import Profiles, Result 6 | 7 | 8 | def test_profiles(pydrill_instance): 9 | """ 10 | :type pydrill_instance: pydrill.client.PyDrill 11 | """ 12 | profiles = pydrill_instance.profiles() 13 | assert type(profiles) == Profiles 14 | assert profiles.response.status_code == 200 15 | assert type(profiles.running_queries) == type([]) 16 | assert type(profiles.finished_queries) == type([]) 17 | 18 | 19 | @responses.activate 20 | def test_profile(pydrill_instance, pydrill_url): 21 | """ 22 | :type pydrill_instance: pydrill.client.PyDrill 23 | """ 24 | query_id = uuid.uuid4() 25 | 26 | responses.add(**{ 27 | 'method': responses.GET, 28 | 'url': "{0}/{1}".format(pydrill_url, 'profiles/{0}.json'.format(query_id)), 29 | 'content_type': 'application/json', 30 | 'status': 200, 31 | }) 32 | 33 | result = pydrill_instance.profile(query_id=query_id) 34 | assert type(result) == Result 35 | assert result.response.status_code == 200 36 | 37 | 38 | @responses.activate 39 | def test_profile_cancel(pydrill_instance, pydrill_url): 40 | """ 41 | :type pydrill_instance: pydrill.client.PyDrill 42 | """ 43 | query_id = uuid.uuid4() 44 | 45 | responses.add(**{ 46 | 'method': responses.GET, 47 | 'url': "{0}/{1}".format(pydrill_url, 'profiles/cancel/{0}'.format(query_id)), 48 | 'content_type': 'application/json', 49 | 'status': 200, 50 | }) 51 | 52 | result = pydrill_instance.profile_cancel(query_id=query_id) 53 | assert type(result) == Result 54 | assert result.response.status_code == 200 55 | -------------------------------------------------------------------------------- /tests/test_pydrill_setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import responses 5 | from pydrill.exceptions import TransportError 6 | 7 | 8 | def test_transport_host(pydrill_instance): 9 | """ 10 | :type pydrill_instance: pydrill.client.PyDrill 11 | """ 12 | assert pydrill_instance.transport.host == os.environ.get('PYDRILL_HOST', 'localhost') 13 | 14 | 15 | def test_transport_port(pydrill_instance): 16 | """ 17 | :type pydrill_instance: pydrill.client.PyDrill 18 | """ 19 | assert pydrill_instance.transport.port == os.environ.get('PYDRILL_PORT', 8047) 20 | 21 | 22 | def test_is_active(pydrill_instance): 23 | """ 24 | :type pydrill_instance: pydrill.client.PyDrill 25 | """ 26 | assert pydrill_instance.is_active() is True 27 | 28 | 29 | @responses.activate 30 | def test_is_not_active_404(pydrill_instance): 31 | """ 32 | :type pydrill_instance: pydrill.client.PyDrill 33 | """ 34 | responses.add(**{ 35 | 'method': responses.HEAD, 36 | 'url': 'http://localhost:8047/', 37 | 'content_type': 'application/json', 38 | 'status': 404, 39 | }) 40 | assert pydrill_instance.is_active() is False 41 | 42 | 43 | @responses.activate 44 | def test_is_not_active_500(pydrill_instance, pydrill_url): 45 | """ 46 | :type pydrill_instance: pydrill.client.PyDrill 47 | """ 48 | responses.add(**{ 49 | 'method': responses.HEAD, 50 | 'url': pydrill_url, 51 | 'content_type': 'application/json', 52 | 'status': 500, 53 | }) 54 | assert pydrill_instance.is_active() is False 55 | 56 | 57 | @responses.activate 58 | def test_is_not_active_201(pydrill_instance, pydrill_url): 59 | """ 60 | :type pydrill_instance: pydrill.client.PyDrill 61 | """ 62 | responses.add(**{ 63 | 'method': responses.HEAD, 64 | 'url': pydrill_url, 65 | 'content_type': 'application/json', 66 | 'status': 201, 67 | }) 68 | assert pydrill_instance.is_active() is False 69 | 70 | 71 | def test_is_not_active_timeout(pydrill_instance): 72 | """ 73 | :type pydrill_instance: pydrill.client.PyDrill 74 | """ 75 | try: 76 | pydrill_instance.perform_request('HEAD', '/', params={'request_timeout': 0.00001}) 77 | except TransportError as e: 78 | assert e.status_code == e.args[0] 79 | assert e.error == e.args[1] 80 | assert e.info == e.args[2] 81 | assert str(e) 82 | else: 83 | assert False 84 | 85 | # TODO: create more tests checking other params. 86 | -------------------------------------------------------------------------------- /tests/test_query.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import pytest 4 | from pydrill.exceptions import QueryError, ImproperlyConfigured 5 | 6 | 7 | def test_select_employee(pydrill_instance): 8 | """ 9 | :type pydrill_instance: pydrill.client.PyDrill 10 | """ 11 | sql = "SELECT * FROM cp.`employee.json` ORDER BY salary DESC LIMIT 1" 12 | expected_result = {'columns': ['birth_date', 'department_id', 'education_level', 'employee_id', 'first_name', 13 | 'full_name', 'gender', 'hire_date', 'last_name', 'management_role', 'marital_status', 14 | 'position_id', 'position_title', 'salary', 'store_id', 'supervisor_id'], 15 | 'rows': [{'last_name': 'Nowmer', 'marital_status': 'S', 'management_role': 'Senior Management', 16 | 'store_id': '0', 'position_id': '1', 'birth_date': '1961-08-26', 17 | 'education_level': 'Graduate Degree', 'gender': 'F', 'supervisor_id': '0', 18 | 'salary': '80000.0', 'department_id': '1', 'position_title': 'President', 19 | 'full_name': 'Sheri Nowmer', 'hire_date': '1994-12-01 00:00:00.0', 20 | 'first_name': 'Sheri', 'employee_id': '1'}]} 21 | 22 | result = pydrill_instance.query(sql=sql) 23 | 24 | assert result.response.status_code == 200 25 | assert result.data == expected_result 26 | 27 | 28 | def test_select_iterator(pydrill_instance): 29 | """ 30 | :type pydrill_instance: pydrill.client.PyDrill 31 | """ 32 | sql = "SELECT * FROM cp.`employee.json` ORDER BY salary DESC LIMIT 1" 33 | 34 | for row in pydrill_instance.query(sql=sql): 35 | assert type(row) is dict 36 | 37 | 38 | def test_select_pandas(pydrill_instance): 39 | """ 40 | :type pydrill_instance: pydrill.client.PyDrill 41 | """ 42 | sql = "SELECT * FROM cp.`employee.json` ORDER BY salary DESC LIMIT 1" 43 | 44 | with pytest.raises(ImproperlyConfigured): 45 | df = pydrill_instance.query(sql=sql).to_dataframe() 46 | 47 | 48 | def test_select_without_sql(pydrill_instance): 49 | """ 50 | :type pydrill_instance: pydrill.client.PyDrill 51 | """ 52 | sql = "" 53 | try: 54 | result = pydrill_instance.query(sql=sql) 55 | except QueryError as e: 56 | assert e 57 | 58 | 59 | def test_plan_for_select_employee(pydrill_instance): 60 | """ 61 | :type pydrill_instance: pydrill.client.PyDrill 62 | """ 63 | sql = "SELECT * FROM cp.`employee.json` ORDER BY salary DESC LIMIT 1" 64 | result = pydrill_instance.plan(sql=sql) 65 | assert result.response.status_code == 200 66 | 67 | 68 | # TODO: create more tests with other queries. 69 | -------------------------------------------------------------------------------- /tests/test_storage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import pytest 4 | import responses 5 | 6 | from pydrill.client.result import Result 7 | 8 | 9 | def test_storage(pydrill_instance): 10 | """ 11 | :type pydrill_instance: pydrill.client.PyDrill 12 | """ 13 | result = pydrill_instance.storage() 14 | assert type(result) == Result 15 | assert result.response.status_code == 200 16 | 17 | 18 | @pytest.mark.parametrize('name', [ 19 | 'mongo', 'hive', 'cp', 'json', 'dfs', 's3', 'kudu', 'hbase' 20 | ]) 21 | def test_storage_detail(pydrill_instance, name): 22 | """ 23 | :type pydrill_instance: pydrill.client.PyDrill 24 | """ 25 | result = pydrill_instance.storage_detail(name=name) 26 | assert type(result) == Result 27 | assert result.response.status_code == 200 28 | 29 | 30 | @pytest.mark.parametrize('name', [ 31 | 'mongo', 'hive', 'cp', 'json', 'dfs', 's3', 'kudu', 'hbase' 32 | ]) 33 | @pytest.mark.parametrize('value', [ 34 | False, True 35 | ]) 36 | @responses.activate 37 | def test_storage_enable(pydrill_instance, pydrill_url, name, value): 38 | """ 39 | :type pydrill_instance: pydrill.client.PyDrill 40 | """ 41 | responses.add(**{ 42 | 'method': responses.GET, 43 | 'url': '{0}/{1}'.format(pydrill_url, 'storage/{0}/enable/{1}'.format(name, 'true' if value else 'false')), 44 | 'content_type': 'application/json', 45 | 'status': 200, 46 | }) 47 | result = pydrill_instance.storage_enable(name=name, value=value) 48 | assert type(result) == Result 49 | assert result.response.status_code == 200 50 | 51 | 52 | @pytest.mark.parametrize('name', [ 53 | 'mongo', 'hive', 'cp', 'dfs', 's3', 'kudu', 'hbase' 54 | ]) 55 | def test_storage_update(pydrill_instance, name): 56 | """ 57 | :type pydrill_instance: pydrill.client.PyDrill 58 | """ 59 | result = pydrill_instance.storage_detail(name=name) 60 | result = pydrill_instance.storage_update(name=name, config=result.data) 61 | assert type(result) == Result 62 | assert result.response.status_code == 200 63 | 64 | 65 | @pytest.mark.parametrize('name', [ 66 | 'mongo', 'hive', 'cp', 'json', 'dfs', 's3', 'kudu', 'hbase' 67 | ]) 68 | @responses.activate 69 | def test_storage_delete(pydrill_instance, pydrill_url, name): 70 | """ 71 | :type pydrill_instance: pydrill.client.PyDrill 72 | """ 73 | responses.add(**{ 74 | 'method': responses.DELETE, 75 | 'url': '{0}/{1}'.format(pydrill_url, 'storage/{0}.json'.format(name)), 76 | 'content_type': 'application/json', 77 | 'status': 200, 78 | }) 79 | result = pydrill_instance.storage_delete(name=name) 80 | assert type(result) == Result 81 | assert result.response.status_code == 200 82 | -------------------------------------------------------------------------------- /tests/test_threads.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from pydrill.client.result import Result 4 | 5 | 6 | def test_threads(pydrill_instance): 7 | """ 8 | :type pydrill_instance: pydrill.client.PyDrill 9 | """ 10 | threads = pydrill_instance.threads() 11 | assert type(threads) == Result 12 | assert threads.response.status_code == 200 13 | assert type(threads.data) == type(u"") 14 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = check-isort, check-flake8, python2.6, python2.7, python3.3, python3.4, python3.5, python3.6, pypy 3 | skipsdist = True 4 | 5 | [testenv] 6 | setenv = 7 | PYTHONPATH = {toxinidir}:{toxinidir}/pydrill 8 | passenv = 9 | PYDRILL_HOST 10 | PYDRILL_PORT 11 | 12 | deps = 13 | -rrequirements_testing.txt 14 | 15 | commands = 16 | py.test -vv tests/ {posargs:--cov=pydrill --cov-report=term-missing} 17 | 18 | [testenv:check-isort] 19 | # isort configurations are located in setup.cfg 20 | deps = isort==4.2.2 21 | commands = isort -rc -c pydrill 22 | 23 | [testenv:run-isort] 24 | # isort configurations are located in setup.cfg 25 | deps = isort==4.2.2 26 | commands = isort -rc pydrill 27 | 28 | [testenv:check-flake8] 29 | # flake8 configurations are located in setup.cfg 30 | deps = flake8==2.5.1 31 | commands = flake8 pydrill 32 | 33 | 34 | ; If you want to make tox run the tests with the same versions, create a 35 | ; requirements.txt with the pinned versions and uncomment the following lines: 36 | ; deps = 37 | ; -r{toxinidir}/requirements.txt 38 | -------------------------------------------------------------------------------- /travis_pypi_setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """Update encrypted deploy password in Travis config file 4 | """ 5 | 6 | 7 | from __future__ import print_function 8 | import base64 9 | import json 10 | import os 11 | from getpass import getpass 12 | import yaml 13 | from cryptography.hazmat.primitives.serialization import load_pem_public_key 14 | from cryptography.hazmat.backends import default_backend 15 | from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 16 | 17 | 18 | try: 19 | from urllib import urlopen 20 | except: 21 | from urllib.request import urlopen 22 | 23 | 24 | GITHUB_REPO = 'PythonicNinja/pydrill' 25 | TRAVIS_CONFIG_FILE = os.path.join( 26 | os.path.dirname(os.path.abspath(__file__)), '.travis.yml') 27 | 28 | 29 | def load_key(pubkey): 30 | """Load public RSA key, with work-around for keys using 31 | incorrect header/footer format. 32 | 33 | Read more about RSA encryption with cryptography: 34 | https://cryptography.io/latest/hazmat/primitives/asymmetric/rsa/ 35 | """ 36 | try: 37 | return load_pem_public_key(pubkey.encode(), default_backend()) 38 | except ValueError: 39 | # workaround for https://github.com/travis-ci/travis-api/issues/196 40 | pubkey = pubkey.replace('BEGIN RSA', 'BEGIN').replace('END RSA', 'END') 41 | return load_pem_public_key(pubkey.encode(), default_backend()) 42 | 43 | 44 | def encrypt(pubkey, password): 45 | """Encrypt password using given RSA public key and encode it with base64. 46 | 47 | The encrypted password can only be decrypted by someone with the 48 | private key (in this case, only Travis). 49 | """ 50 | key = load_key(pubkey) 51 | encrypted_password = key.encrypt(password, PKCS1v15()) 52 | return base64.b64encode(encrypted_password) 53 | 54 | 55 | def fetch_public_key(repo): 56 | """Download RSA public key Travis will use for this repo. 57 | 58 | Travis API docs: http://docs.travis-ci.com/api/#repository-keys 59 | """ 60 | keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo) 61 | data = json.loads(urlopen(keyurl).read().decode()) 62 | if 'key' not in data: 63 | errmsg = "Could not find public key for repo: {}.\n".format(repo) 64 | errmsg += "Have you already added your GitHub repo to Travis?" 65 | raise ValueError(errmsg) 66 | return data['key'] 67 | 68 | 69 | def prepend_line(filepath, line): 70 | """Rewrite a file adding a line to its beginning. 71 | """ 72 | with open(filepath) as f: 73 | lines = f.readlines() 74 | 75 | lines.insert(0, line) 76 | 77 | with open(filepath, 'w') as f: 78 | f.writelines(lines) 79 | 80 | 81 | def load_yaml_config(filepath): 82 | with open(filepath) as f: 83 | return yaml.load(f) 84 | 85 | 86 | def save_yaml_config(filepath, config): 87 | with open(filepath, 'w') as f: 88 | yaml.dump(config, f, default_flow_style=False) 89 | 90 | 91 | def update_travis_deploy_password(encrypted_password): 92 | """Update the deploy section of the .travis.yml file 93 | to use the given encrypted password. 94 | """ 95 | config = load_yaml_config(TRAVIS_CONFIG_FILE) 96 | 97 | config['deploy']['password'] = dict(secure=encrypted_password) 98 | 99 | save_yaml_config(TRAVIS_CONFIG_FILE, config) 100 | 101 | line = ('# This file was autogenerated and will overwrite' 102 | ' each time you run travis_pypi_setup.py\n') 103 | prepend_line(TRAVIS_CONFIG_FILE, line) 104 | 105 | 106 | def main(args): 107 | public_key = fetch_public_key(args.repo) 108 | password = args.password or getpass('PyPI password: ') 109 | update_travis_deploy_password(encrypt(public_key, password.encode())) 110 | print("Wrote encrypted password to .travis.yml -- you're ready to deploy") 111 | 112 | 113 | if '__main__' == __name__: 114 | import argparse 115 | parser = argparse.ArgumentParser(description=__doc__) 116 | parser.add_argument('--repo', default=GITHUB_REPO, 117 | help='GitHub repo (default: %s)' % GITHUB_REPO) 118 | parser.add_argument('--password', 119 | help='PyPI password (will prompt if not provided)') 120 | 121 | args = parser.parse_args() 122 | main(args) 123 | --------------------------------------------------------------------------------