├── .editorconfig ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── NOTICE.md ├── README.md ├── api_docs ├── Makefile ├── authors.rst ├── conf.py ├── contributing.rst ├── history.rst ├── index.rst ├── installation.rst ├── make.bat ├── modules.rst ├── readme.rst ├── stackhut.barrister.rst ├── stackhut.rst └── usage.rst ├── docs ├── Makefile ├── conf.py ├── creating_service │ ├── app_structure.rst │ ├── index.rst │ ├── service_runtime.rst │ ├── service_structure.rst │ └── stackhut_toolkit.rst ├── examples │ └── index.rst ├── getting_started │ ├── index.rst │ ├── installation.rst │ ├── tutorial.rst │ └── tutorial_use.rst ├── index.rst ├── introduction.rst ├── misc │ ├── credits.rst │ ├── faq.rst │ └── index.rst ├── service_mgmt │ ├── hosting.rst │ ├── index.rst │ ├── management.rst │ └── testing.rst └── using_service │ ├── client_libs.rst │ ├── general.rst │ ├── index.rst │ └── json_rpc.rst ├── pip_release.sh ├── readme_pip.rst ├── requirements.txt ├── screencasts ├── create_ascii.sh ├── use_ascii.sh └── use_script.md ├── setup.cfg ├── setup.py ├── setup_freeze.py ├── stackhut.py ├── stackhut_toolkit ├── __init__.py ├── __main__.py ├── builder.py ├── commands.py ├── common │ ├── __init__.py │ ├── barrister │ │ ├── __init__.py │ │ ├── cythonplex3 │ │ │ ├── Actions.pxd │ │ │ ├── Actions.py │ │ │ ├── DFA.py │ │ │ ├── Errors.py │ │ │ ├── Lexicons.py │ │ │ ├── Machines.py │ │ │ ├── Regexps.py │ │ │ ├── Scanners.pxd │ │ │ ├── Scanners.py │ │ │ ├── Timing.py │ │ │ ├── Traditional.py │ │ │ ├── Transitions.py │ │ │ └── __init__.py │ │ ├── exceptions.py │ │ ├── parser.py │ │ └── runtime.py │ ├── commands.py │ ├── config.py │ ├── exceptions.py │ ├── runtime │ │ ├── __init__.py │ │ ├── backends.py │ │ ├── rpc.py │ │ ├── runner.py │ │ └── runtime_server.py │ └── utils.py ├── manager.py ├── res │ ├── scaffold │ │ ├── common │ │ │ ├── .gitignore │ │ │ ├── Hutfile.yaml │ │ │ ├── README.md │ │ │ ├── api.idl │ │ │ └── test_request.json │ │ ├── nodejs │ │ │ └── app.js │ │ └── python │ │ │ ├── app.py │ │ │ └── requirements.txt │ ├── shims │ │ ├── nodejs │ │ │ ├── runner.js │ │ │ └── stackhut.js │ │ └── python │ │ │ ├── runner.py │ │ │ └── stackhut.py │ └── templates │ │ ├── Docker-runtime.txt │ │ ├── Dockerfile-baseos.txt │ │ ├── Dockerfile-service.txt │ │ └── Dockerfile-stack.txt ├── run_commands.py └── toolkit_utils.py ├── tests ├── __init__.py └── test_toolkit.py ├── toolkit.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .stackhut.log 2 | *.json 3 | !test_request.json 4 | run_result 5 | .stackhut 6 | .DS_Store 7 | 8 | # ide 9 | *.iml 10 | .idea 11 | 12 | # Byte-compiled / optimized / DLL files 13 | __pycache__/ 14 | *.py[cod] 15 | 16 | # C extensions 17 | *.so 18 | 19 | # Distribution / packaging 20 | .Python 21 | env/ 22 | build/ 23 | develop-eggs/ 24 | dist/ 25 | downloads/ 26 | eggs/ 27 | .eggs/ 28 | lib/ 29 | lib64/ 30 | parts/ 31 | sdist/ 32 | var/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *,cover 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | api_docs/_build 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | 72 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | python: 6 | - "3.4" 7 | - "2.7" 8 | 9 | # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 10 | install: pip install -r requirements.txt 11 | 12 | # command to run tests, e.g. python setup.py test 13 | script: python setup.py test -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. 5 | 6 | You can contribute in many ways: 7 | 8 | Types of Contributions 9 | ---------------------- 10 | 11 | ### Report Bugs 12 | 13 | Report bugs at . 14 | 15 | If you are reporting a bug, please include: 16 | 17 | - Your operating system name and version. 18 | - Any details about your local setup that might be helpful in troubleshooting. 19 | - Detailed steps to reproduce the bug. 20 | 21 | ### Fix Bugs 22 | 23 | Look through the GitHub issues for bugs. Anything tagged with "bug" is open to whoever wants to implement it. 24 | 25 | ### Implement Features 26 | 27 | Look through the GitHub issues for features. Anything tagged with "feature" is open to whoever wants to implement it. 28 | 29 | ### Write Documentation 30 | 31 | StackHut Console Tool could always use more documentation, whether as part of the official StackHut Console Tool docs, in docstrings, or even on the web in blog posts, articles, and such. 32 | 33 | ### Submit Feedback 34 | 35 | The best way to send feedback is to file an issue at . 36 | 37 | If you are proposing a feature: 38 | 39 | - Explain in detail how it would work. 40 | - Keep the scope as narrow as possible, to make it easier to implement. 41 | - Remember that this is a volunteer-driven project, and that contributions are welcome :) 42 | 43 | Get Started! 44 | ------------ 45 | 46 | Ready to contribute? Here's how to set up stackhut for local development. 47 | 48 | 1. Fork the stackhut-toolkit repo on GitHub. 49 | 2. Clone your fork locally: 50 | 51 | $ git clone git@github.com:your_name_here/stackhut-toolkit.git 52 | 53 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development: 54 | 55 | $ mkvirtualenv stackhut-toolkit 56 | $ cd stackhut-toolkit/ 57 | $ python setup.py develop 58 | 59 | 4. Create a branch for local development: 60 | 61 | $ git checkout -b name-of-your-bugfix-or-feature 62 | 63 | Now you can make your changes locally. 64 | 65 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox: 66 | 67 | $ flake8 stackhut tests 68 | $ python setup.py test 69 | $ tox 70 | 71 | There's an included `Makefile` that includes `test` and `test-all` commands that automate this. To get flake8 and tox, just pip install them into your virtualenv. 72 | 73 | 6. Commit your changes and push your branch to GitHub: 74 | 75 | $ git add . 76 | $ git commit -m "Your detailed description of your changes." 77 | $ git push origin name-of-your-bugfix-or-feature 78 | 79 | 7. Submit a pull request through the GitHub website. 80 | 81 | Pull Request Guidelines 82 | ----------------------- 83 | 84 | Before you submit a pull request, check that it meets these guidelines: 85 | 86 | 1. The pull request should include tests. 87 | 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. 88 | 3. The pull request should work for Python 3.4. Check and make sure that the tests pass for all supported Python versions. 89 | 90 | Tips 91 | ---- 92 | 93 | To run a subset of tests: 94 | 95 | $ python -m unittest tests.test_stackhut 96 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include stackhut_toolkit/res * 8 | 9 | recursive-include tests * 10 | recursive-exclude * __pycache__ 11 | recursive-exclude * *.py[co] 12 | 13 | # recursive-include docs *.rst conf.py Makefile make.bat 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build api_docs clean 2 | 3 | help: 4 | @echo "clean - remove all build, test, coverage and Python artifacts" 5 | @echo "clean-build - remove build artifacts" 6 | @echo "clean-pyc - remove Python file artifacts" 7 | @echo "clean-test - remove test and coverage artifacts" 8 | @echo "lint - check style with flake8" 9 | @echo "test - run tests quickly with the default Python" 10 | @echo "test-all - run tests on every Python version with tox" 11 | @echo "coverage - check code coverage quickly with the default Python" 12 | @echo "api_docs - generate Sphinx HTML documentation, including API api_docs" 13 | @echo "release - package and upload a release" 14 | @echo "dist - package" 15 | @echo "install - install the package to the active Python's site-packages" 16 | 17 | clean: clean-build clean-pyc clean-test 18 | 19 | clean-build: 20 | rm -fr build/ 21 | rm -fr dist/ 22 | rm -fr stackhut.build/ 23 | rm -fr stackhut.dist/ 24 | rm -fr .eggs/ 25 | rm -fr *-wheel *-sdist 26 | find . -name '*.egg-info' -exec rm -fr {} + 27 | find . -name '*.egg' -exec rm -f {} + 28 | 29 | clean-pyc: 30 | find . -name '*.pyc' -exec rm -f {} + 31 | find . -name '*.pyo' -exec rm -f {} + 32 | find . -name '*~' -exec rm -f {} + 33 | find . -name '__pycache__' -exec rm -fr {} + 34 | 35 | clean-test: 36 | rm -fr .tox/ 37 | rm -f .coverage 38 | rm -fr htmlcov/ 39 | 40 | lint: 41 | -flake8 stackhut tests 42 | 43 | test: 44 | python3 setup.py test 45 | 46 | test-all: 47 | tox 48 | 49 | coverage: 50 | coverage run --source stackhut setup.py test 51 | coverage report -m 52 | coverage html 53 | xdg-open htmlcov/index.html 54 | 55 | api_docs: 56 | rm -f api_docs/stackhut.rst 57 | rm -f api_docs/modules.rst 58 | sphinx-apidoc -o api_docs/ stackhut 59 | $(MAKE) -C api_docs clean 60 | $(MAKE) -C api_docs html 61 | xdg-open api_docs/_build/html/index.html 62 | 63 | release: dist 64 | git push 65 | git push --tags 66 | twine upload dist/* 67 | 68 | dist: clean 69 | python3 setup.py bdist_wheel 70 | ls -l dist 71 | 72 | install: clean 73 | python3 setup.py install 74 | 75 | 76 | -------------------------------------------------------------------------------- /NOTICE.md: -------------------------------------------------------------------------------- 1 | Copyright 2015 StackHut Ltd. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | --- 16 | 17 | ## Libraries Used (to be properly updated...) - 18 | * [Barrister](http://barrister.bitmechanic.com/) 19 | * 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StackHut Platform 2 | ## Deploy classes as Microservices 3 | 4 | **Notice:** *The classical python "StackHut Toolkit" is deprecated, in favor of a more robust, compiled implementation. This here is kept for historical purpose, and unfortunately will not receive any significant work/love going forwards. We will however have a new open-source implementation that you can follow available shortly.* 5 | 6 | [![image](https://img.shields.io/pypi/v/stackhut.svg)](https://pypi.python.org/pypi/stackhut) 7 | [![Documentation Status](https://readthedocs.org/projects/stackhut/badge/?version=latest)](http://stackhut.readthedocs.org/?badge=latest) 8 | 9 | StackHut is a platform to develop and deploy microservices without writing any server-logic. It takes a regular class (in Python or JavaScript for now), a YAML file describing your stack, and deploys a microservice whose functions can be called natively in other languages, or through REST. StackHut is pure Software Defined Infrastructure, and abstracts away web-frameworks, servers, and infrastructure entirely. 10 | 11 | The `stackhut` command tool provides CLI functionality into creating, running, and deploying StackHut images. Available to download in both source and binary form for Linux and OSX (Win support on the way). Free software under the Apache license. 12 | 13 | Happy hacking! :) 14 | 15 | --- 16 | 17 | ## Related Repos 18 | ### Client Libraries 19 | * [NodeJS/ES6](https://github.com/stackhut/client-node) 20 | * [Python3](https://github.com/stackhut/client-python) 21 | 22 | ### Samples 23 | 24 | #### Basic Examples 25 | * [demo-python](https://github.com/stackhut/demo-python) - Basic Python 3 Example 26 | * [demo-python-persistent](https://github.com/stackhut/demo-python-persistent) - Basic Python 3 Example with state 27 | * [demo-nodejs](https://github.com/stackhut/demo-nodejs) - Basic NodeJS/ES6 Example 28 | * [demo-nodejs-persistent](https://github.com/stackhut/demo-nodejs-persistent) - Basic NodeJS/ES6 Example with State 29 | 30 | #### More Complex Examples 31 | 32 | * [pdf-tools](https://github.com/stackhut/pdf-tools) - PDF Rendering-as-a-Service 33 | * [image-process](https://github.com/stackhut/image-process) - Image Processing-as-a-Service using ImageMagick 34 | * [web-tools](https://github.com/stackhut/web-tools) - Remote Web-Rendering and tooling using PhantomJS 35 | * [media-download](https://github.com/stackhut/media-download) - youtube-dl-as-Service to download media from online sites (may be blocked from services from time to time) 36 | * [t-shirt-aaS](https://github.com/stackhut/t-shirt-aaS) - Not yet live 37 | 38 | All of these services are all live and running under the `stackhut` namespace, i.e. `stackhut/servicename`, and can be called and tried out. 39 | You can call them using any of the client libraries or `curl` as described in the [StackHut documentation](http://stackhut.readthedocs.org/). 40 | 41 | ## Useful Links 42 | 43 | * Homepage: https://www.stackhut.com 44 | * User Manual & Docs: http://docs.stackhut.com 45 | * [GitHub Issues](https://github.com/stackhut/stackhut/issues) 46 | * [GitHub Wiki](https://github.com/stackhut/stackhut/wiki) 47 | 48 | --- 49 | 50 | # Getting started 51 | ## Installing the toolkit 52 | 53 | All releases can be found on this repo's [release page](https://github.com/stackhut/stackhut/releases). Detailed install instructions can be found in the [User Manual](http://docs.stackhut.com/getting_started/installation.html). 54 | 55 | _Note_ - StackHut requires [Docker](www.docker.com) to be installed - on OSX/Windows download [Docker Toolbox](https://www.docker.com/docker-toolbox) and on Linux we recommend using your distro version. 56 | 57 | ### Binary/Standalone Install 58 | 59 | You can download a standalone executable for Linux and OSX. 60 | * On OSX there are three binary install methods: 61 | * Using brew - `brew install stackhut/stackhut/toolkit` (a 3rd-party tap you can also upgrade with `brew upgrade stackhut/stackhut/toolkit` - make sure you have an up-to-date brew with `brew update`) 62 | * Download and run the latest `.pkg` file from the [releases page](https://github.com/stackhut/stackhut/releases) (standalone that you can remove simply by `sudo rm -rf /usr/local/bin/stackhut /usr/local/opt/stackhut`) 63 | * Download and unpack the portable `.txz`-archive from the [releases page](https://github.com/stackhut/stackhut/releases) 64 | * Linux 65 | * Download and unpack the portable `.txz`-archive from the [releases page](https://github.com/stackhut/stackhut/releases) 66 | 67 | ### Source Install 68 | 69 | Alternatively, source builds are always available using `pip` and are the recommended way to install if you already have Python 3: 70 | 71 | * On OSX, `brew install python3; pip3 install stackhut --user` (or just `pip3 install stackhut --user` if you already have Python 3), 72 | * On Linux, `pip3 install stackhut --user` (you may need to install Python 3 first - it's installed by default on newer distros). 73 | 74 | ### Developer Install 75 | 76 | Want to run the latest code from Git? Awesome! 77 | * clone this repo - `git clone git@github.com:StackHut/stackhut-toolkit.git` 78 | * `cd stackhut-toolkit` 79 | * `pip3 install -r ./requirements.txt` (Install the dependencies) 80 | * `python3 ./setup.py develop --user` (you may need to re-run this command occasionally after updating from git) 81 | 82 | 83 | 84 | ### Follow the tutorial 85 | 86 | An in-depth tutorial showing how to create, call and deploy a simple service can be found in the [User Manual](http://docs.stackhut.com/getting_started/tutorial.html). 87 | 88 | --- 89 | 90 | ## Contributing 91 | 92 | Contributions are welcome, and greatly appreciated! Every little bit helps us approach the NoOps dream, and credit will always be given :) 93 | 94 | Please see [CONTRIBUTING.md](./CONTRIBUTING.md) for more info. 95 | -------------------------------------------------------------------------------- /api_docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/stackhut.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/stackhut.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/stackhut" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/stackhut" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /api_docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /api_docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /api_docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /api_docs/index.rst: -------------------------------------------------------------------------------- 1 | .. stackhut documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to StackHut Console Tool's documentation! 7 | ====================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | installation 16 | usage 17 | contributing 18 | authors 19 | history 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | 28 | -------------------------------------------------------------------------------- /api_docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | At the command line:: 6 | 7 | $ easy_install stackhut 8 | 9 | Or, if you have virtualenvwrapper installed:: 10 | 11 | $ mkvirtualenv stackhut-tool 12 | $ pip install stackhut 13 | -------------------------------------------------------------------------------- /api_docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\stackhut.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\stackhut.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /api_docs/modules.rst: -------------------------------------------------------------------------------- 1 | stackhut 2 | ======== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | stackhut 8 | -------------------------------------------------------------------------------- /api_docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /api_docs/stackhut.barrister.rst: -------------------------------------------------------------------------------- 1 | stackhut.common.barrister package 2 | ========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | stackhut.common.barrister.runtime module 8 | --------------------------------- 9 | 10 | .. automodule:: stackhut.common.barrister.runtime 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: stackhut.common.barrister 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /api_docs/stackhut.rst: -------------------------------------------------------------------------------- 1 | stackhut package 2 | ================ 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | stackhut.common.barrister 10 | 11 | Submodules 12 | ---------- 13 | 14 | stackhut.build_commands module 15 | ------------------------------ 16 | 17 | .. automodule:: stackhut.build_commands 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | stackhut.toolkit.commands module 23 | ------------------------ 24 | 25 | .. automodule:: stackhut.toolkit.commands 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | 30 | stackhut.run_command module 31 | --------------------------- 32 | 33 | .. automodule:: stackhut.run_command 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | stackhut.utils module 39 | --------------------- 40 | 41 | .. automodule:: stackhut.utils 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | 46 | 47 | Module contents 48 | --------------- 49 | 50 | .. automodule:: stackhut 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | -------------------------------------------------------------------------------- /api_docs/usage.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Usage 3 | ======== 4 | 5 | To use StackHut Console Tool in a project:: 6 | 7 | import stackhut 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/StackHut.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/StackHut.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/StackHut" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/StackHut" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/creating_service/app_structure.rst: -------------------------------------------------------------------------------- 1 | .. _creating_app: 2 | 3 | Service App Code 4 | ================ 5 | 6 | In this section we exaplin how to write you code to deploy it as a microservice - we have endeavoured to make this part as simple as possible, so you can get on with the business of writing your business logic normally and not worry about infrastructure, servers, or hosting. 7 | 8 | 9 | .. _creating_app_idl: 10 | 11 | Interface Definition (``api.idl``) 12 | ---------------------------------- 13 | 14 | **TODO** 15 | 16 | See :ref:`tutorial_create`. 17 | 18 | This is based on the `Barrister RPC project `_, the format of which is described in the `project documentation `_. 19 | 20 | 21 | 22 | App Code 23 | -------- 24 | 25 | **TODO** 26 | 27 | 28 | 29 | 30 | .. _creating_app_auth: 31 | 32 | Authorisation 33 | ^^^^^^^^^^^^^ 34 | 35 | By default if an ``auth`` object exists in the request (see :ref:`using_general_auth`) it's checked by the API layer and only routed to the service if it's valid for **any** StackHut user, regardless if the service is public or private. Authentication is handled at the API layer. 36 | 37 | A service must determine if the authenticated request should be allowed or not, and the StackHut runtime (see :ref:`creating_runtime`) has several functions to aid this authorisation, 38 | 39 | * ``stackhut.get_stackhut_user()`` returns the username user to make the request, 40 | * ``stackhut.is_author()`` returns a boolean indicating if the author's Stackhut account was used to make the request. This can help to restrict a private service's interfaces to the service author only and any associated API tokens. 41 | 42 | We have separated authentication from authorisation is to aid deploying internal StackHut clusters where there will be many users owning services, in this case we believe that individual service authorisation is better handled within the service itself according to internal policies. 43 | 44 | .. note:: However we are working on simplying this scheme of users on the hosted platform 45 | 46 | .. _creating_app_lang: 47 | 48 | Language Specific Notes 49 | ----------------------- 50 | 51 | 52 | Python 3.x 53 | ^^^^^^^^^^ 54 | 55 | As mentioned above all calls to the server are blocking (although handled in multiple threads). 56 | 57 | There are two methods to deal with this on the Python side. 58 | Firstly Python has a good threading library that works very well when used with blocking IO calls. 59 | Secondly Python 3.5 will introduce ``async`` and ``await`` type-functionality as seen in C# that can be used to interleave these calls. 60 | 61 | 62 | Node.js / ES6 63 | ^^^^^^^^^^^^^ 64 | 65 | The Node.js story is more complex as StackHut is primarily a request->response system exhibited through functions as entrypoints. This conflicts with the callback-based model of Node at present. 66 | 67 | However things are looking much better with both ES6 and ES7 on the horizon. 68 | StackHut's Node support is based on the latest io.js with support for ES6, and promises in particular (`here's `_ is a good intro). 69 | 70 | .. note:: We currently utilise `io.js v3 `_ to provide a compatible version of Node.js with ES6 features. These projects have now remerged and we will follow io.js in moving to the Node.js project accordingly. 71 | 72 | The StackHut runtime is promise-based on Node, with each call returning a promise than resolves on completion. 73 | 74 | Similarly the main entrypoints to a service are also promise-based. The StackHut runner expects each entrypoint to return a promise that resolves on completion of the service request. 75 | This gives you two choices when implementing a service function depending on if it's a synchronous or a callback-based asynchronous service. 76 | 77 | 78 | Synchronous Services 79 | """""""""""""""""""" 80 | 81 | Write your function as normal and simply wrap the result in a Promise.resolve(). 82 | 83 | .. code-block:: js 84 | 85 | add(x, y) { 86 | let res = x + y; 87 | return Promise.resolve(res); 88 | } 89 | 90 | 91 | Asynchronous Services 92 | """"""""""""""""""""" 93 | 94 | Wrap your service block in a single promise that is returned to the system. Within this block write your normal code and call with ``resolve`` or ``reject`` as required on completion. This method interacts nicely with new promise-based and legacy callback-based async code. 95 | 96 | .. code-block:: js 97 | 98 | asyncAdd(x, y) { 99 | return new Promise(function(resolve, reject) { 100 | someAsyncCall(x, y) 101 | .then(function (res) { 102 | resolve(res); 103 | 104 | }) 105 | }) 106 | } 107 | 108 | 109 | .. note:: As we support regular ES6 with node packages, feel free to add any helpers libraries to your ``package.json`` to ease writing async services, i.e. `co `_. 110 | 111 | Similar to Python 3.5, ``async`` and ``await`` are coming with ES7 and will provide a better model for async code that will be easier to integrate with StackHut. 112 | 113 | 114 | -------------------------------------------------------------------------------- /docs/creating_service/index.rst: -------------------------------------------------------------------------------- 1 | .. _creating_index: 2 | 3 | ****************** 4 | Creating a Service 5 | ****************** 6 | 7 | In this section we describe the individual components used when creating a StackHut service, including the command-line (CLI) Toolkit and the StackHut runtime library. 8 | 9 | .. toctree:: 10 | 11 | stackhut_toolkit 12 | service_structure 13 | app_structure 14 | service_runtime 15 | 16 | -------------------------------------------------------------------------------- /docs/creating_service/service_runtime.rst: -------------------------------------------------------------------------------- 1 | .. _creating_runtime: 2 | 3 | StackHut Runtime Library 4 | ======================== 5 | 6 | When running a StackHut service there are many common functions you may wish to perform and interact with the host environment, be it running locally on your machine or when hosted on the StackHut platform. 7 | 8 | In this section we describe the StackHut library full of common functionality you can use in your service to make the process a lot easier. 9 | 10 | 11 | Usage 12 | ----- 13 | 14 | The StackHut library is available on all supported language stacks. 15 | It exposes a common set of functions across them all for interacting with the hosting platform and the wider world when running a service. 16 | 17 | In most languages you simply import the ``stackhut`` module within your service and use the functions directly. (If you used ``stackhut init`` this will already be done within the created skeleton service.) 18 | 19 | API 20 | --- 21 | 22 | download_file 23 | ^^^^^^^^^^^^^ 24 | 25 | .. code-block:: python 26 | 27 | stackhut.download_file(url, fname) 28 | 29 | Downloads a file from the given ``url`` into the working directory. If ``fname`` is provided will rename the download file to this value, else will use the original filename. 30 | 31 | Returns the filename of the downloaded file. 32 | 33 | get_file 34 | ^^^^^^^^ 35 | 36 | .. code-block:: python 37 | 38 | stackhut.get_file(key) 39 | 40 | This function is used to download files uploaded using the ``/files`` endpoint (see :ref:`using_general_files`). It securly downloads the file refernced by the given ``key`` into the working directory and returns the filename. 41 | 42 | 43 | put_file 44 | ^^^^^^^^ 45 | 46 | .. code-block:: python 47 | 48 | stackhut.put_file(fname, make_public) 49 | 50 | This function uploads the file referenced by ``fname`` in the service working directory to cloud storage (S3) where it can be downloaded by yourself or others. 51 | ``make_public`` is an optional boolean that triggers whether the uploaded file is made accessible as a public URL, by default this is True. 52 | 53 | Returns the URL of the uploaded file. 54 | 55 | 56 | get_stackhut_user 57 | ^^^^^^^^^^^^^^^^^ 58 | 59 | .. code-block:: python 60 | 61 | stackhut.get_stackhut_user() 62 | 63 | Returns the authenticated StackHut username of the request originator as a string, or ``null`` if not present. This has been authenticated by the server and can be used securely to know who made the request. 64 | 65 | 66 | is_author 67 | ^^^^^^^^^ 68 | 69 | .. code-block:: python 70 | 71 | stackhut.is_author() 72 | 73 | Returns ``True`` if the authenticated StackHut user who made the request is the service author, authenticated either using the author hash or via a valid token tied to that account (see :ref:`using_general_auth`). 74 | 75 | 76 | run_command 77 | ^^^^^^^^^^^ 78 | 79 | .. code-block:: python 80 | 81 | stackhut.run_command(cmd, stdin) 82 | 83 | Runs the command specified by ``cmd`` as an external process and waits for completeion. ``stdin`` is an optional string that, when specified, will be used as the STDIN to the command. 84 | 85 | Function waits for the subprocess to complete and returns STDOUT as a string. 86 | 87 | 88 | General Notes 89 | ------------- 90 | 91 | All functions are blocking by default, however each call runs in a separate thread so you may make multiple calls without blocking the runtime library itself. However dealing with the associate blocking on the client side is a matter for your particular service and language stack. This issue is described further in :ref:`creating_app`. 92 | 93 | 94 | -------------------------------------------------------------------------------- /docs/creating_service/service_structure.rst: -------------------------------------------------------------------------------- 1 | .. _creating_structure: 2 | 3 | Service Project Structure 4 | ========================= 5 | 6 | Introduction 7 | ------------ 8 | In this section we describe the files created when you initialise a StackHut project using ``stackhut init baseos language`` (e.g. ``stackhut init fedora python``). 9 | 10 | .. _creating_structure_hutfile: 11 | 12 | Hutfile 13 | ------- 14 | 15 | The ``Hutfile`` is a YAML file that describes the configuration of your StackHut service. It has several required fields that describe the project and let you control the packaging of your service. 16 | 17 | Here's a sample, 18 | 19 | .. code-block:: yaml 20 | 21 | # Name for the service - we recommend using snake-case 22 | name: demo-python 23 | 24 | # Service description 25 | description: Awesome service in python 26 | 27 | # GitHub source repo link (optional) 28 | github_url: www.github.com/StackHut/demo-python 29 | 30 | # The service dependencies, in terms of the base OS and language runtime 31 | baseos: fedora 32 | stack: python 33 | 34 | # Persist the service between requests 35 | persistent: False 36 | 37 | # Restrict service access to authenticated users 38 | private: False 39 | 40 | Let's go through all the fields. 41 | 42 | ``name`` 43 | ^^^^^^^^ 44 | 45 | *Required* 46 | 47 | The name of the service - this does not need to be unique. We recommend keeping this the same as your project-s name in source control (e.g. on GitHub) and using snake-case. 48 | 49 | 50 | ``github_url`` 51 | ^^^^^^^^^^^^^^ 52 | 53 | *Optional* 54 | 55 | A link to the repository on GitHub - we use this to create links on the service homepage to the issue tracker and source, if available. 56 | 57 | ``description`` 58 | ^^^^^^^^^^^^^^^ 59 | 60 | *Optional* 61 | 62 | A short text description of the service. 63 | 64 | ``baseos`` 65 | ^^^^^^^^^^ 66 | 67 | *Required* 68 | 69 | The base OS to use when creating an image. Currently we support, 70 | 71 | ====== =========== 72 | Name Description 73 | ====== =========== 74 | fedora `Fedora 22 `_ 75 | ubuntu `Ubuntu Linux `_ 76 | debian `Debian Linux `_ 77 | 78 | ====== =========== 79 | 80 | 81 | .. _creating_structure_hutfile_stack: 82 | 83 | ``stack`` 84 | ^^^^^^^^^ 85 | 86 | *Required* 87 | 88 | 89 | The base language stack to use when creating an image. Currently we support, 90 | 91 | ======= =========== 92 | Name Description 93 | ======= =========== 94 | python `Python 3 `_ 95 | nodejs `Node.js `_ (implemented using `io.js `_) 96 | ======= =========== 97 | 98 | .. note:: Currently we only support one language stack per service (although you could use this to call anothor language you've bundled into the image) 99 | 100 | .. note:: If your chosen language stack has a package manager, e.g. ``pip``, ``npm``, etc., you can fill out the package file, e.g. ``requirements.txt``, ``package.json``, etc., and it will be installed automatically within your image. 101 | 102 | 103 | ``private`` 104 | ^^^^^^^^^^^ 105 | 106 | *Required* 107 | 108 | Boolean indicating if the service is private. If true only requests by an authenticated StackHut user are permitted. If false the service is public and requests to the service can be made by anyone. 109 | 110 | ``persistent`` 111 | ^^^^^^^^^^^^^^ 112 | 113 | *Required* 114 | 115 | Indicates whether the service persists between request or is terminated and restart on demand on each new request. Setting a service as persistent decreases response times and can hold state between requests, however will require more resources. 116 | 117 | ``files`` 118 | ^^^^^^^^^ 119 | 120 | *Optional* 121 | 122 | A list of files and directories within the project directory to include and bundle alongside the image. Useful for specifying resource files and binaries, for instance. 123 | 124 | See ``_ for an example. 125 | 126 | ``os_deps`` 127 | ^^^^^^^^^^^ 128 | 129 | *Optional* 130 | 131 | A list of OS packages you wish to bundle with your service, i.e. those installable via ``yum``, or ``apt-get``, depending on your ``baseos``. You may need to check with your choosing base OS repository to find the names of the packages and their versions. 132 | 133 | .. note:: you can also install language specific packages using your language package manager as described in the :ref:`creating_structure_hutfile_stack` subsection above. 134 | 135 | See ``_ for an example. 136 | 137 | 138 | ``docker_cmds`` 139 | ^^^^^^^^^^^^^^^ 140 | 141 | *Optional* 142 | 143 | If the default workflow is not flexible enough, you can specify any additional scripting commands as a list within this section. They will be run when building your container and follow the `Dockerfile builder specification `_. 144 | 145 | See ``_ for an example. 146 | 147 | -------------------------------------------------------------------------------- /docs/creating_service/stackhut_toolkit.rst: -------------------------------------------------------------------------------- 1 | .. _creating_toolkit: 2 | 3 | StackHut Toolkit 4 | ================ 5 | 6 | 7 | .. Introduction 8 | .. ------------ 9 | 10 | The StackHut Toolkit is used to create, test, deploy, and maintain your services hosted on StackHut. 11 | It provides a range of commands used to interact with your code and the StackHut servers. 12 | 13 | Getting Started 14 | ^^^^^^^^^^^^^^^ 15 | 16 | Firstly, install the Toolkit and requirements by following the instructions in :ref:`getting_started_installation`. 17 | 18 | **Quick Install** 19 | 20 | .. code:: bash 21 | 22 | sudo pip3 install stackhut 23 | 24 | **Upgrade** 25 | 26 | .. code:: bash 27 | 28 | sudo pip3 install --upgrade stackhut 29 | 30 | .. note:: Things move pretty quickly on the Toolkit so if you find an error try upgrading first to see if it's been fixed. Thanks! 31 | 32 | Usage 33 | ----- 34 | 35 | Having installed the Toolkit just make sure it's accessible from your path 36 | 37 | .. code:: bash 38 | 39 | $ stackhut -V 40 | > stackhut 0.5.4 41 | 42 | Now that's done, you may wish to run through the tutorial at :ref:`tutorial_create`. 43 | 44 | We've tried to make the Toolkit as easy to use as possible and it follows the ``git`` and ``docker`` command based model, e.g. ``stackhut init``, etc. 45 | You can find the list of commands and options available by running, 46 | 47 | .. code:: bash 48 | 49 | $ stackhut --help 50 | 51 | .. note:: Enable verbose mode to view more debug output using ``stackhut -v``. 52 | 53 | Commands 54 | -------- 55 | 56 | In this section we'll go over the main commands supported by the Toolkit and explain their use in helping you to build, test, deploy and maintain your services locally and in the cloud. 57 | 58 | Help for any command can be displayed by running, 59 | 60 | .. code:: bash 61 | 62 | $ stackhut command --help 63 | 64 | 65 | ``login`` 66 | ^^^^^^^^^ 67 | 68 | .. code:: bash 69 | 70 | $ stackhut login 71 | 72 | This command logins you into the StackHut platform using your GitHub username and StackHut password. 73 | You need to be logged to build and deploy service (this is so we know how to correctly name the image when building the service), 74 | 75 | .. note:: Your details are stored in a user-only readable file at ``$HOME/.stackhut.cfg``. 76 | 77 | 78 | ``logout`` 79 | ^^^^^^^^^^ 80 | 81 | .. code:: bash 82 | 83 | $ stackhut logout 84 | 85 | Logs you out of the StackHut platform. 86 | 87 | ``info`` 88 | ^^^^^^^^ 89 | 90 | .. code:: bash 91 | 92 | $ stackhut info 93 | 94 | Displays information regarding the Toolkit version, Docker version, and current logged-in user. 95 | 96 | ``init`` 97 | ^^^^^^^^ 98 | .. code:: bash 99 | 100 | $ stackhut init baseos stack [--no-git] 101 | 102 | ============ =========== 103 | Option Description 104 | ============ =========== 105 | ``baseos`` The base operating system to use, e.g. fedora, alpine, ubuntu, etc. 106 | ``stack`` The default language stack to use, e.g. python, nodejs, etc. 107 | ``--no-git`` Disables creating a git repo as part of the init process 108 | ============ =========== 109 | 110 | Initialises a new StackHut project in the current directory using the specified base Operating System and language stack. This creates a working skeleton project you can modify to rapidly build your own service. 111 | 112 | By default it creates a service in your stack that has a single ``add`` function already specified. The initial project is comprised of the following files, 113 | 114 | * a minimal ``Hutfile`` (see :ref:`creating_structure_hutfile`), 115 | * an ``api.idl`` inteface definition file (see :ref:`creating_app_idl`), 116 | * an ``app.py`` application file (or app.js, etc.), 117 | * a ``README.md`` markdown file, 118 | * a ``test_request.json`` test file to simulate requests to your service, 119 | * an empty packages file for your chosen language stack (e.g. ``requirements.txt`` for Python, or ``package.json`` for Node, etc.). 120 | 121 | The ``init`` command also creates a git repo and commits the files be default, to disable this behaviour use the ``--no-git`` flag. 122 | 123 | 124 | ``build`` 125 | ^^^^^^^^^ 126 | 127 | .. code:: bash 128 | 129 | $ stackhut build [--force] 130 | 131 | ============ =========== 132 | Option Description 133 | ============ =========== 134 | ``--force`` Forces the build to occur even if no file changes 135 | ============ =========== 136 | 137 | Builds the image so that it may be tested locally or deployed to the cloud. This command is usually unneeded as both the ``run`` and ``deploy`` commands run a build if needed. 138 | 139 | Building a service involves, 140 | * setting up the base OS and the language stack, 141 | * installing all OS and language packages as specified in the `Hutfile`, 142 | * copying across all files referenced in the `Hutfile`, 143 | * installing the StackHut control runner, 144 | * running any auxiliary commands as specified in the `Hutfile`. 145 | 146 | Building can be time-consuming so is performed on an as-needed basis by detecting changes to the files referenced from the `Hutfile`. If this fails, or perhaps you're installing software from across the network as part of the build, you may wish to force the build to occur by passing the ``--force`` flag. 147 | 148 | 149 | ``runcontainer`` 150 | ^^^^^^^^^^^^^^^^ 151 | 152 | .. code:: bash 153 | 154 | $ stackhut runcontainer [--force] 155 | 156 | ================ =========== 157 | Option Description 158 | ================ =========== 159 | ``--force`` Forces build before run 160 | ================ =========== 161 | 162 | 163 | Builds the image and and hosts the service locally on ``http://localhost:4001``. You can test the service either using the client-libaries or by ``curl``-ing the ``test_request.json`` file to the local server, as described in :ref:`using_index`. 164 | 165 | Upon running this command the Toolkit will build the image (if required) and run the service within the container. This is exactly the same code as will be run on the hosted StackHut platform so you can be sure that if it works locally it will work in the cloud. Output from running this request is placed in the ``run_result`` directory, with the JSON response object in ``run_result\response.json``. 166 | 167 | 168 | ``runhost`` 169 | ^^^^^^^^^^^ 170 | 171 | .. code:: bash 172 | 173 | $ stackhut runhost request_file 174 | 175 | 176 | The ``runcontainer`` command builds and runs an full image - we make every effort to cache and reduce the time this process takes but you may find it still imposes a delay when testing quick changes. 177 | To this end we provide the ``runhost`` command - it runs your service immediately using your host operating system and installed dependencies instead. 178 | 179 | As with the ``run`` command it hosts the service locally for use with the client-libraries and writes the response into ``run_result``. 180 | 181 | This can be a useful way to setup a quick feedback loop, but we recommend using the ``runcontainer`` command in most cases as it will test your entire service and dependencies using the same code as on the server. 182 | Furthermore it can be easier to setup the dependencies for the service in the container and they'll be isolated from the main host OS. 183 | 184 | .. note:: ``runhost`` will not install any dependencies from the `Hutfile` for you and you will have to manually set these up if needed. 185 | 186 | ``deploy`` 187 | ^^^^^^^^^^ 188 | 189 | .. code:: bash 190 | 191 | $ stackhut deploy [--force] 192 | 193 | ================ =========== 194 | Option Description 195 | ================ =========== 196 | ``--force`` Forces build before deploy 197 | ================ =========== 198 | 199 | .. ``--no-build`` Deploy only, do not build or push image first 200 | 201 | The deploy command packages and uploads your project to the StackHut platform where it's build remotely and then deployed live under the service address ``username/servicename`` and can be called from ``https://api.stackhut.com/run``. 202 | Deployment requires that you have an account at StackHut and are logged in using the command line tool. 203 | 204 | .. If you've already deployed the image and just want to update the service metadata, e.g. the description, README, API docs, etc., you can run ``deploy`` with the ``--no-build`` flag and it will skip the full deploy - a much quicker operation. 205 | 206 | -------------------------------------------------------------------------------- /docs/examples/index.rst: -------------------------------------------------------------------------------- 1 | .. _examples_index: 2 | 3 | ******** 4 | Examples 5 | ******** 6 | 7 | This section contains a list of more complex services we've built using StackHut. They are all `open-source `_ and each project has detailed documentation describing the service's creation. 8 | 9 | Each service showcases different features/techniques that can be used to build StackHut services for your application. 10 | 11 | 12 | PDF Tools 13 | ========= 14 | 15 | * Documented Source - http://www.github.com/StackHut/pdf-tools 16 | * Service Homepage - http://www.stackhut.com/#/u/stackhut/pdf-tools 17 | 18 | ``pdf-tools`` is a StackHut service that converts PDFs to various formats, including to text and multiple image formats. This Python-based service demonstrates the following features, 19 | 20 | * File manipulation 21 | * OS dependencies 22 | * *Shelling*-out to binary/command-line tools from within a service 23 | 24 | 25 | Image Process 26 | ============= 27 | 28 | * Documented Source - http://www.github.com/StackHut/image-process 29 | * Service Homepage - http://www.stackhut.com/#/u/stackhut/image-process 30 | 31 | ``image-process`` is a StackHut service that performs various image manipulation tasks and can be used to generate *memes*. This Python-based service demonstrates the following features, 32 | 33 | * File manipulation 34 | * OS and language dependencies 35 | * Embedding and accessing resource files within a service 36 | 37 | 38 | ========= 39 | Web Tools 40 | ========= 41 | 42 | * Documented Source - http://www.github.com/StackHut/web-tools 43 | * Service Homepage - http://www.stackhut.com/#/u/stackhut/web-tools 44 | 45 | ``web-tools`` is a StackHut service that performs several web-developer related functions. This service wraps up the `PhantomJS `_ and `Selenium `_ libraries to provide a headless web-browser. 46 | 47 | It uses these to provide a screen-shotting service that works for JS-heavy sites but will be expanded in the future. This Python-based service demonstrates the following features, 48 | 49 | * OS dependencies 50 | * Embedding and accessing resource files within a service 51 | * Custom binaries and dependencies 52 | * Running arbitrary `Docker build `_ commands 53 | 54 | -------------------------------------------------------------------------------- /docs/getting_started/index.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started_index: 2 | 3 | *************** 4 | Getting Started 5 | *************** 6 | 7 | In this section we describe how you can install the StackHut toolkit, and use it in the :ref:`getting_started_tutorial` to create, deploy and call a simple service at scale in a few minutes. 8 | 9 | .. toctree:: 10 | 11 | installation 12 | tutorial 13 | 14 | -------------------------------------------------------------------------------- /docs/getting_started/installation.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started_installation: 2 | 3 | Platform Installation 4 | ===================== 5 | 6 | This page describes installing the command-line StackHut Toolkit so you can rapidly develop, test, and deploy your services. 7 | 8 | All releases can be found on the stackhut repo's `release page `_. 9 | 10 | Binary/Standalone Install 11 | ------------------------- 12 | 13 | You can download a standalone executable for Linux and OSX. 14 | 15 | OSX 16 | ^^^ 17 | 18 | On OSX there are three binary install methods: 19 | 20 | * Use brew, ``brew install stackhut/stackhut/toolkit`` 21 | 22 | This is a 3rd-party tap from which you can upgrade using ``brew upgrade stackhut/stackhut/toolkit``, 23 | 24 | .. note:: Make sure you have an up-to-date brew with ``brew update``. 25 | 26 | * OR - download and run the latest ``.pkg`` file from the `release page `_. This is a standalone package you can remove with ``sudo rm -rf /usr/local/bin/stackhut /usr/local/opt/stackhut``, 27 | 28 | * OR - download and unpack the portable ``.txz``-archive from the `release page `_. 29 | 30 | Linux 31 | ^^^^^ 32 | 33 | * Download and unpack the portable ``.txz``-archive from the `release page `_ 34 | 35 | Source Install 36 | -------------- 37 | 38 | Alternatively, source builds are always available using ``pip`` and are the recommended way to install if you already have Python 3: 39 | 40 | OSX 41 | ^^^ 42 | 43 | ``brew install python3; pip3 install stackhut --user`` (or just ``pip3 install stackhut --user`` if you already have Python 3), 44 | 45 | Linux 46 | ^^^^^ 47 | 48 | ``pip3 install stackhut --user`` (you may need to install Python 3 first - it's installed by default on newer distros). 49 | 50 | .. note:: Using the ``--user`` flag will install in the user's ``$HOME`` directory. However on OSX you'll have to manually add ``~/Libraries/Python/3.4/bin`` to your ``$PATH``. Omitting the ``--user`` flag it will require ``sudo`` and install globally instead. 51 | 52 | 53 | Developer Install 54 | ----------------- 55 | 56 | Want to run the latest code from Git? Awesome! 57 | 58 | #) ``git clone git@github.com:stackhut/stackhut.git`` (Clone the repo) 59 | #) ``cd stackhut`` 60 | #) ``pip3 install -r ./requirements.txt`` (Install the dependencies) 61 | #) ``python3 ./setup.py develop --user`` 62 | 63 | .. note:: You may need to re-run the last command occasionally after updating from ``git``. 64 | 65 | 66 | Install Notes 67 | ------------- 68 | 69 | 70 | Requirements 71 | ^^^^^^^^^^^^ 72 | 73 | * `Docker `_ to develop and test services locally. On OSX/Windows download `Docker Toolbox `_ and on Linux we recommend using your distro version. 74 | 75 | .. note:: Currently we support Linux and OSX - with Windows support launching shortly. 76 | 77 | Upgrading 78 | ^^^^^^^^^ 79 | 80 | Development on the StackHut Toolkit moves pretty fast, so if you find a bug it may be worth updating first before reporting an issue. On the binary releases it's just as easy as re-installing the latest package. For the source releases, just run ``pip3 install --upgrade stackhut``. 81 | 82 | 83 | Next Steps 84 | ---------- 85 | 86 | An in-depth tutorial showing how to create, call and deploy a simple service can be found in the :ref:`getting_started_tutorial`. 87 | 88 | 89 | -------------------------------------------------------------------------------- /docs/getting_started/tutorial_use.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial_use: 2 | 3 | Tutorial - Using a Service 4 | ========================== 5 | 6 | In this tutorial presents a quick overview of how to access a StackHut service from within your application, whether it is written in Python, client/server-side JS, Ruby, .NET, and more. 7 | 8 | Overview 9 | -------- 10 | 11 | All StackHut services can be accessed and consumed via a direct HTTP POST request. On receiving a request, the StackHut host platform will route the request on demand to the required service to complete it. 12 | The whole StackHut infrastructure is abstracted away from your service code, from its point of view it's simply executing a function call. 13 | 14 | 15 | It can then be accessed in the cloud via `JSON-RPC `_ transported over a HTTP(S) POST request. 16 | To make it easier to call and use StackHut services we have started building client-libraries in several lanauges. They are described further in :ref:`using_client_libs`, and currently exist for Python and JavaScript. 17 | 18 | However it's always possible to contsruct the JSON-RPC request yourself in any lanauge to consume a StackHut service - thankfully JSON-RPC is a very simple protocol, as shown in :ref:`using_json_rpc`, and this is much simpler than it sounds! 19 | 20 | 21 | Selecting a service 22 | ------------------- 23 | 24 | You can find all kinds of services, for instance, video encoding, compression, compilation, web scraping, and more, hosted at the `StackHut repository `_. 25 | 26 | Sercices are prefixed by their author, such as ``stackhut/demo-python``. We can view the documentation and API for this service on its `homepage `_, it has 2 methods, ``add`` and ``multiply``. 27 | 28 | 29 | Calling a service 30 | ----------------- 31 | 32 | For this tutorial we'll use the ``demo-python`` service created in above (if you didn't create one you can use ``stackhut/demo-python`` instead). We'll use the Python 3.x client library (described in :ref:`using_client_libs`) to call this service. 33 | 34 | First we'll create a ``SHService`` object to reference the service, 35 | 36 | .. code-block:: python 37 | 38 | import stackhut_client as client 39 | service = client.SHService('stackhut', 'demo-python') 40 | 41 | where ``stackhut`` is the service author (replace with your own service name), and ``demo-python`` is the service name. Now we have the service we can just call the methods on the ``Default`` interface, 42 | 43 | .. code-block:: python 44 | 45 | service.Default.add(1, 2) 46 | >> 3 47 | service.Default.multiply(2, 3) 48 | >> 6 49 | 50 | 51 | Thanks for reading this tutorial - you can find more information on calling services in :ref:`using_index`. Further detailes decribed how we built and can call more complex services, such as a web-scraper, or an image-processor, can be found in :ref:`examples`. 52 | 53 | .. Want to develop a StackHut cloud API or fork an existing service? Read :ref:`tutorial_create` to get going - we can't wait to see what you come up with. 54 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. StackHut documentation master file, created by 2 | sphinx-quickstart on Sun Jul 12 14:37:33 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to StackHut's documentation! 7 | ==================================== 8 | 9 | .. note:: The classical python "StackHut Toolkit" is deprecated, in favor of a more robust, compiled implementation. This here is kept for historical purpose, and unfortunately will not receive any significant work/love going forwards. We will however have a new open-source implementation that you can follow available shortly. 10 | 11 | Thank you for browsing our documentation. Please let us know at docs@stackhut.com if anything is unclear or you find a mistake. Better still, submit a pull-request using the link at the top-right. 12 | 13 | Check out :ref:`getting_started_index` for a quick guide to both using and creating APIs on StackHut. 14 | 15 | Contents: 16 | 17 | .. toctree:: 18 | :maxdepth: 2 19 | 20 | introduction 21 | getting_started/index 22 | examples/index 23 | creating_service/index 24 | using_service/index 25 | service_mgmt/index 26 | misc/index 27 | 28 | 29 | Indices and tables 30 | ================== 31 | 32 | * :ref:`genindex` 33 | * :ref:`search` 34 | 35 | .. * :ref:`modindex` 36 | 37 | -------------------------------------------------------------------------------- /docs/introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ************ 3 | 4 | StackHut is a way to deploy and run code in the cloud without worrying about a traditional server. 5 | 6 | You write a normal class in Python or JavaScript, specify your stack (operating system, language, and any dependencies), and specify the interface into your code. When you deploy it, it is provisioned and replicated on a Kubernetes cluster and can be called directly through an HTTP POST request, or through JSON-RPC, from anywhere (even client-side JavaScript.) 7 | 8 | This means you can run functions and fully-fledged applications without deploying code to a traditional server. You can focus on the business logic, instead of spending time with server/API logic. 9 | -------------------------------------------------------------------------------- /docs/misc/credits.rst: -------------------------------------------------------------------------------- 1 | Credits 2 | ======= 3 | 4 | StackHut relies on several pieces of third-party code that we would like to give thanks to: 5 | 6 | * `Barrister RPC `_ 7 | * `Docker `_ 8 | * `Kubernetes `_ 9 | * `Python `_ 10 | * and more... 11 | 12 | Thanks to all of you :) 13 | The StackHut team. 14 | 15 | -------------------------------------------------------------------------------- /docs/misc/faq.rst: -------------------------------------------------------------------------------- 1 | Frequently Asked Questions 2 | ========================== 3 | 4 | What is it 5 | ^^^^^^^^^^ 6 | 7 | StackHut let's you take your local code, including binaries, resourres and dependicies, and push them into a cloud API that is accesible via functions calls over JSON-RPC. 8 | 9 | Why would I use this 10 | ^^^^^^^^^^^^^^^^^^^^ 11 | Deploying code to the cloud is hard for several 12 | 13 | What use-cases do you envision? 14 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 15 | 16 | 17 | Can I use it now? 18 | ^^^^^^^^^^^^^^^^^ 19 | 20 | Are there more docs on the inteface deinfinal file (IDL)? 21 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 22 | 23 | What are the technical limitations? 24 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 25 | 26 | What are the practical limitations? 27 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 28 | 29 | Do you charge for this? 30 | ^^^^^^^^^^^^^^^^^^^^^^^ 31 | 32 | What's going on behind the scenes? 33 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 34 | 35 | What languages do you support? 36 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 37 | 38 | What's it written in? 39 | ^^^^^^^^^^^^^^^^^^^^^ 40 | 41 | Is it open-source? 42 | ^^^^^^^^^^^^^^^^^^ 43 | 44 | 45 | -------------------------------------------------------------------------------- /docs/misc/index.rst: -------------------------------------------------------------------------------- 1 | .. _misc_index: 2 | 3 | ************* 4 | Miscellaneous 5 | ************* 6 | .. other stuff - misc, contributing, authors, history 7 | 8 | 9 | .. toctree:: 10 | 11 | credits 12 | .. faq 13 | 14 | -------------------------------------------------------------------------------- /docs/service_mgmt/hosting.rst: -------------------------------------------------------------------------------- 1 | Service Hosting 2 | =============== 3 | 4 | Introduction 5 | ------------ 6 | 7 | **TODO** 8 | 9 | 10 | Usage 11 | ----- 12 | 13 | **TODO** 14 | 15 | 16 | -------------------------------------------------------------------------------- /docs/service_mgmt/index.rst: -------------------------------------------------------------------------------- 1 | .. _mgmt_index: 2 | 3 | ****************** 4 | Service Management 5 | ****************** 6 | 7 | In this section we desctibe serveral of the remaining issues outside of both creating and using serices, including the hosting your services in the ccloud and locally, service testing, management and scaling, updating, and more. 8 | 9 | .. toctree:: 10 | 11 | hosting 12 | testing 13 | management 14 | 15 | -------------------------------------------------------------------------------- /docs/service_mgmt/management.rst: -------------------------------------------------------------------------------- 1 | Service Management 2 | ================== 3 | 4 | Introduction 5 | ------------ 6 | 7 | **TODO** 8 | 9 | 10 | Usage 11 | ----- 12 | 13 | **TODO** 14 | 15 | -------------------------------------------------------------------------------- /docs/service_mgmt/testing.rst: -------------------------------------------------------------------------------- 1 | Service Testing 2 | =============== 3 | 4 | Introduction 5 | ------------ 6 | 7 | **TODO** 8 | 9 | 10 | Usage 11 | ----- 12 | 13 | **TODO** 14 | 15 | -------------------------------------------------------------------------------- /docs/using_service/client_libs.rst: -------------------------------------------------------------------------------- 1 | .. _using_client_libs: 2 | 3 | Client-side Libraries 4 | ===================== 5 | 6 | Client-side libraries are have been/or are under development for the following platforms, please feel free to add support for your favourite language if not present, it's quite easy! 7 | 8 | ============= ============================== =========== 9 | Langauge Install Source Code 10 | ============= ============================== =========== 11 | Python pip3 install stackhut-client http://www.github.com/StackHut/client-python 12 | JavaScript npm install stackhut-client http://www.github.com/StackHut/client-node 13 | Ruby *under development* 14 | PHP *under development* 15 | Java/JVM *under development* 16 | C#/.NET *under development* 17 | ============= ============================== =========== 18 | 19 | 20 | These libraries abstract away the entire JSON-RPC mechanism and make it as easy as calling a function to utilise a StackHut service. They marshal the data, collect the response, handling error messages along the way, and check the validity of the message before it's sent. For example, in the following code we create a service object to use an existing service called `demo-nodejs` by the user `stackhut`. Using this object we can call any functions on any interfaces exposed by the hosted `stackhut/demo-nodejs` service, 21 | 22 | 23 | .. code-block:: python 24 | 25 | import stackhut_client as client 26 | service = client.SHService('stackhut', 'web-tools') 27 | result = service.Default.renderWebpage('http://www.stackhut.com', 1024, 768) 28 | print(result) 29 | >> http://stackhut-files.s3.amazonaws.com/stackhut/downloads/a77d49f6-af7d-4007-8630-f6f443de7680/5c77d73b-9c8c-4850-84eb-9196b19fb545/screen.png 30 | 31 | 32 | Client libraries API 33 | -------------------- 34 | 35 | The the general behaviour of the client libraries is similar in all languages and we describe it below using the Python client-library as a reference. 36 | There are 3 main classes in the library, 37 | 38 | SHService 39 | ^^^^^^^^^ 40 | 41 | The main class representing a single StackHut service. It takes several parameters on construction, where those in square brackets are optional, 42 | 43 | .. code:: python 44 | 45 | import stackhut_client as client 46 | client.SHService(author, service_name, [service_version], [auth], [host]) 47 | 48 | * author - The author of the service 49 | * service_name - The service name 50 | * version - The specific version of the service (is `latest` if left blank) 51 | * auth - An `SHAuth` object used to authenticate requests for private services 52 | * host - URL for the StackHut API server, can be set to point to local servers during development, is `https://api.stackhut.com` if left blank 53 | 54 | To make a remote call, just call the interface and method name on the service object, e.g., 55 | 56 | .. code:: python 57 | 58 | result = service.Interface.method(params, ...) 59 | 60 | 61 | SHAuth 62 | ^^^^^^ 63 | 64 | An optional class used to authenticate requests to a service, passed to a service on construction, 65 | 66 | .. code:: python 67 | 68 | client.SHAuth(user, [hash], [token]) 69 | 70 | * user - Username of a registered StackHut user 71 | * hash - Hash of the user's password (you can find this in ~/.stackhut.cfg). Be careful not to use in public-accessible code 72 | * token - A valid API token created for the user 73 | 74 | One of `hash` or `token` must be present in the `auth` object to authorise a request by the given user. 75 | 76 | SHError 77 | ^^^^^^^ 78 | 79 | Returned in the event of a remote service error as an exception/error depending on the specific client library. 80 | 81 | The object has 3 parameters, 82 | 83 | * code - A JSON-RPC error code 84 | * message - A string describing the error 85 | * data - An optional object that may contain additional structured data for handling the error 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /docs/using_service/general.rst: -------------------------------------------------------------------------------- 1 | .. _using_general: 2 | 3 | General Usage Notes 4 | =================== 5 | 6 | StackHut services can be accessed easily either through client side libraries, or directly using the underlying JSON+HTTP protocol. Either way there are several pints to note to make communicating with your services as easy and efficient as possible. 7 | 8 | 9 | .. _using_general_files: 10 | 11 | Files 12 | ----- 13 | 14 | Often you will want to pass a file from your code to be processed by a StackHut service, for instance when processing a video or converting a PDF. 15 | 16 | Currently files must to be uploaded separately to a online location from where they can be retrieved by the service over HTTP, for instance AWS S3, rather than embedded within the remote message. 17 | 18 | To aid this we provide an endpoint, ``https://api.stackhut.com/files``, that you can ``POST`` to obtain a signed URL. The content of this ``POST`` request is a single JSON object, ``auth``, that contains both the username and either a ``hash`` or ``token`` element, similar to ``SHAuth`` above, 19 | 20 | .. code-block:: JSON 21 | 22 | { 23 | "auth" : { 24 | "username" : "USERNAME", 25 | "hash" : "HASH" 26 | } 27 | } 28 | 29 | 30 | The request returns a single object containing both, ``key``, a signed URL to which you may ``PUT`` a resource, and ``key``, a key to be passed to the StackHut service identifying the file. Within the StackHut service, a service author can call `stackhut.get_file(key)` from the StackHut runtime library (see :ref:`creating_runtime`) to download the file to the working directory. 31 | We recognise this is an extra step and are working hard to remove this limitation. File handling using this endpoint is not supported in the client libraries at present. 32 | 33 | Result files can are automatically placed onto S3 for easy retrieval by clients although can be uploaded elsewhere if required. 34 | 35 | .. _using_general_auth: 36 | 37 | Authentication 38 | -------------- 39 | 40 | Authentication is provided within StackHut - this is used restrict service requests to the service author and explicitly allowed clients only. This can be enabled or disabled on a per-service basis by setting ``private`` to ``True`` or ``False`` respectively within the ``Hutfile.yaml``. 41 | 42 | Authentication requires specifying both the StackHut ``username`` and either the user ``hash`` or a generatated API token that can be shared with clients. A registered StackHut user's ``hash`` can be found by running ``stackhut info`` and can used with private services to authenticate the user. 43 | 44 | .. note:: Do not share your hash or use it directly within code that is run on untrusted devices, e.g. client-side JS. Generate and use an API token instead (however, API Tokens are currently in development. sorry!) 45 | 46 | Authentication is supported within the client-side libs - create an ``SHAuth`` object and pass it to the ``SHService``, as shown in :ref:`using_client_libs`. 47 | This ``auth`` object is added at the top-level to service message, in accordance with the JSON-RPC protocol described in :ref:`using_json_rpc`. You may need to do this manually if creating JSON-RPC messages directly, 48 | 49 | .. code-block:: JSON 50 | 51 | { 52 | "auth" : { 53 | "username" : "$USERNAME", 54 | "hash" : "$HASH" 55 | } 56 | } 57 | 58 | 59 | By default if an ``auth`` object exists in the request it is check by the API layer and only allowed through to the service if the auth object is valid for **any** StackHut user, regardless if the service is public or private. Authorisation is handled within services themselves, as described in :ref:`creating_app_auth`. 60 | 61 | .. note:: To call a public service anonymously it's easiest to just not add an ``auth`` object to the request. We're aware that this can be confusing and are working on a simpler API. 62 | 63 | State 64 | ----- 65 | 66 | Similar to files, we are currently hard at work on providing a standardised solution to handling state within a service - services are generally state-less by default and are fully destroyed and reconstructed between requests. 67 | 68 | However, setting ``persistent: True`` in the ``Hutfile.yaml`` with keep the service alive between calls, and can be used to store state within application memory/local filesystem during the life-cycle of a service. This can be used, for instance, to cache long-running/complex computations or to keep database connections open. However due to the nature of the platform services may be restarted at anytime without warning and we recommend treating this state as ephemeral. 69 | 70 | To store state persistently, outside of the service life-cycle, it's possible to call and store data on an external platform, e.g. a hosted database, on an individual service basis. This is currently outside of StackHut's scope and you'll have to refer to your favourite hosted database documentation to integrate it with your chosen service language. 71 | 72 | Batching 73 | -------- 74 | 75 | We have currently only described StackHut as performing a single request per call, however it's also possible to collect several request and perform them sequentially within a single call. This is termed ``batching`` mode and is easily accomplished in StackHut by simply sending a list of reesusts within the ``req`` object in the call, 76 | 77 | .. code-block:: JSON 78 | 79 | { 80 | "service" : "stackhut/demo-nodejs-state", 81 | "req" : [ 82 | { 83 | "method" : "inc", 84 | "params" : [10] 85 | "id" : 1 86 | }, 87 | { 88 | "method" : "inc", 89 | "params" : [20] 90 | "id" : 2 91 | }, 92 | { 93 | "method" : "getCur", 94 | "params" : [] 95 | "id" : 3 96 | } 97 | ] 98 | } 99 | 100 | These request will all be performed within a single service-call, great for increasing throughput and keeping your external calls over the cloud to StackHut to a minimum. 101 | We have some exciting features planned involving batching that will allow you to setup complex cloud-based processing pipelines easily. 102 | 103 | Batching is not supported in the client libraries at present. 104 | -------------------------------------------------------------------------------- /docs/using_service/index.rst: -------------------------------------------------------------------------------- 1 | .. _using_index: 2 | 3 | *************** 4 | Using a Service 5 | *************** 6 | 7 | In this section we describe how to make use of services within your own code, be it client-side JS, mobile apps, internal systems or server code itself. This can be done using either a client-side library for your chosen language, or by using the underlying JSON+HTTP protocol directly - either way we endeavour to make communicating with your services as easy as possible. 8 | 9 | .. toctree:: 10 | 11 | general 12 | client_libs 13 | json_rpc 14 | 15 | -------------------------------------------------------------------------------- /docs/using_service/json_rpc.rst: -------------------------------------------------------------------------------- 1 | .. _using_json_rpc: 2 | 3 | Direct JSON-RPC Usage 4 | ===================== 5 | 6 | This involves creating JSON-RPC compatible requests on demand, thankfully this is very simple and so it's easy to call your StackHut services from anywhere. 7 | It can then be accessed in the cloud via `JSON-RPC `_ transported over a HTTP(S) POST request. 8 | To make it easier to call and use StackHut services we have started building client-libraries in several languages. They are described further in :ref:`using_client_libs`, and currently exist for Python and JavaScript. 9 | 10 | However it's always possible to construct the JSON-RPC request yourself in any langauge to consume a StackHut service - thankfully JSON-RPC is a very simple protocol, as shown in :ref:`using_json_rpc`, and this is much simpler than it sounds! 11 | 12 | Request: 13 | 14 | .. code-block:: JSON 15 | 16 | { 17 | "service" : "mands/demo-python", 18 | "req" : { 19 | "method" : "add", 20 | "params" : [2, 2] 21 | "id" : 1 22 | } 23 | } 24 | 25 | In the above request, we call the method ``add`` from the StackHut service ``mands/demo-python`` with two parameter, the numbers 2, and 2. 26 | ``params`` is a JSON list that can contain any JSON types, i.e. floats, strings, lists and objects. The types expected by the method are defined by the service and are shown on API section of the `services page `_. The types of the request are checked and an error will be returned if they do not match. 27 | The ``id`` element is optional and will be added automatically if not present. 28 | 29 | We can perform this call by simply sending a HTTP POST request, with content-type ``application/json`` to ``https://api.stackhut.com/run``. Let's save the above example as ``test_request.json`` and demonstrate this using the fantastic tool `httpie `_, 30 | 31 | .. code-block:: bash 32 | 33 | http POST https://api.stackhut.com/run @./test_request.json 34 | 35 | This will make the request and on completion will output a bunch of things to your terminal, including a response body similar to the following, 36 | 37 | .. code-block:: JSON 38 | 39 | { 40 | "response": { 41 | "id": "f335bc80-4289-4599-b655-55341b40bd1a", 42 | "jsonrpc": "2.0", 43 | "result": 4 44 | }, 45 | "taskId": "d2e186d6-e746-4a4f-bb76-b39185e588d5" 46 | } 47 | 48 | The ``response`` object is the JSON-RPC response, containing the return value in the ``result`` field - in this case the number 4 (we created all this to show 2 + 2 = 4 - but now in the cloud! :)). The ``id`` element is also present, to aid linking requests to repsonses, and there is also a top-level ``taskId`` element to uniquely represent this individual request. 49 | 50 | Let's try and call the service method ``multiply`` but with two strings, rather than two numbers as expected. We'll edit the ``test_request.json`` as follows, 51 | 52 | .. code-block:: JSON 53 | 54 | { 55 | "service" : "mands/demo-python", 56 | "req" : { 57 | "method" : "multiply", 58 | "params" : ["two", "three"], 59 | "id" : 1 60 | } 61 | } 62 | 63 | and run,:: 64 | 65 | http POST https://api.stackhut.com/run @./test_request.json 66 | 67 | returning, 68 | 69 | .. code-block:: JSON 70 | 71 | { 72 | "response": { 73 | "error": { 74 | "code": -32602, 75 | "message": "Function 'Default.multiply' invalid param 'x'. 'two' is of type , expected int" 76 | }, 77 | "id": "d15a719a-70e3-4643-87d2-92cb7157bb81", 78 | "jsonrpc": "2.0" 79 | }, 80 | "taskId": "c405cb17-0d57-4aee-804b-ad29edad3000" 81 | } 82 | 83 | 84 | As before we receive a JSON-RPC response object, however this time the ``result`` field has been replaced with an ``error`` field, an object with an error code, a human readable text message, and an optional ``data`` sub-object with further information. You can use this data to handle the error as required within your code 85 | 86 | .. note:: The error codes are as those defined by the `JSON-RPC spec `_. 87 | 88 | We hope this shows how you can call any StackHut service from your code - you may either use an existing JSON-RPC library or roll your own functions to make the request and handle the response respectively. 89 | 90 | 91 | .. Login into StackHut 92 | .. ------------------- 93 | .. __Coming Soon__ - all services are curently free to use and can be accessed anonymously. 94 | 95 | -------------------------------------------------------------------------------- /pip_release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Start a new pip/source release 3 | set -e 4 | 5 | if [[ $# -eq 0 ]] ; then 6 | echo 'Run with new bumpversion {major/minor/patch}' 7 | exit 0 8 | fi 9 | 10 | make clean 11 | make lint 12 | 13 | # TODO - setup tests properly 14 | # test locally 15 | # make test 16 | 17 | # test in venv 18 | # make test-all 19 | 20 | bumpversion $1 21 | make release 22 | make clean 23 | 24 | -------------------------------------------------------------------------------- /readme_pip.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | StackHut Toolkit 3 | ================ 4 | 5 | .. image:: https://img.shields.io/travis/stackhut/stackhut-toolkit.svg 6 | :target: https://travis-ci.org/stackhut/stackhut-toolkit 7 | 8 | .. image:: https://img.shields.io/pypi/v/stackhut.svg 9 | :target: https://pypi.python.org/pypi/stackhut 10 | 11 | `stackhut` command tool provides CLI functionality into creating, running, and deploying StackHut images. Runs on Python 3.4. 12 | 13 | * Homepage: 14 | * Documentation: . 15 | * Source: 16 | * Free software: Apache license 17 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | 3 | -------------------------------------------------------------------------------- /screencasts/create_ascii.sh: -------------------------------------------------------------------------------- 1 | # StackHut allows you to rapidly deploy your code as an API in the cloud 2 | # this screencast shows creating and deploying a simple service on StackHut 3 | 4 | # first make sure we have StackHut (and Docker) installed 5 | sudo pip3 install stackhut > /dev/null 6 | stackhut -V 7 | 8 | # login/register with Docker (to store your StackHut services) 9 | docker login 10 | 11 | # login to StackHut, create an account at https://www.stackhut.com first 12 | stackhut login 13 | stackhut info 14 | 15 | # let's create a Python 3 project using Alpine Linux 16 | mkdir demo-python 17 | cd demo-python 18 | stackhut init alpine python 19 | 20 | # the project is set up, with a Git repo too - aren't we nice... 21 | ls 22 | 23 | # the ``Hutfile`` is a YAML config file regarding our stack and dependencies 24 | alias ccat pygmentize 25 | ccat -l yaml ./Hutfile 26 | 27 | # api.idl describes our service interface with a Java-like syntax 28 | # these entrypoints will be accessible over HTTP 29 | # let's take a look 30 | ccat -l java api.idl 31 | 32 | # we're exposing a single function, "add" 33 | # let's write the signature for a new function, "multiply" 34 | # yes - nano ftw! 35 | nano api.idl 36 | 37 | // multiply 2 integers and return the result 38 | multiply(x int, y int) int 39 | 40 | # now we may write our code 41 | # this lives in app.py (or app.js for JS, etc.) 42 | ccat app.py 43 | 44 | # the service is a Python class with a function for each entrypoint 45 | # "add" has already been implemented, let's write "multiply" 46 | nano app.py 47 | 48 | def multiply(self, x, y): 49 | return x * y 50 | 51 | # we're done coding, and can build and run our service locally 52 | # first let's build our service, i.e. packaging into an image 53 | stackhut build 54 | 55 | # great, and now let's test our service 56 | # the file 'test_request.json' models a HTTP request to the API 57 | # this is a JSON-RPC object that describes the call 58 | ccat ./test_request.json 59 | 60 | # we'll run our service with this file to test "add" 61 | stackhut run test_request.json 62 | 63 | # that worked, we can view the response in ./run_result/response.json 64 | ccat ./run_result/response.json 65 | 66 | # let's modify "test_request.json" to test our multiply function 67 | nano ./test_request.json 68 | stackhut run test_request.json 69 | ccat ./run_result/response.json 70 | 71 | git commit -am "Working service" 72 | 73 | # fantastic, everything is working 74 | # we're now ready to deploy and host the service live on StackHut 75 | # this couldn't be simpler 76 | stackhut deploy 77 | 78 | # great, the API is live, see https://www.stackhut.com/#/u/mands/demo-python 79 | # it can be used via a HTTP POST to https://api.stackhut.com/run 80 | # as shown in http://docs.stackhut.com/getting_started/tutorial_use.html 81 | # thanks for your time - we can't wait to see what you build... 82 | 83 | -------------------------------------------------------------------------------- /screencasts/use_ascii.sh: -------------------------------------------------------------------------------- 1 | # this screencast shows how to use a StackHut service from your code 2 | # whether in Python, JS, Ruby ... or even the shell! 3 | # you can find many services at https://stackhut.com/#/services 4 | # we will be using the service "mands/demo-python" 5 | # (as created in http://docs.stackhut.com/getting_started/tutorial_create.html) 6 | 7 | # let's first view the documentation 8 | open "http://stackhut.com/#/u/mands/demo-python" 9 | 10 | # so it has 2 methods, "add" and "multiply", we can call these via a JSON-RPC HTTP request 11 | # thankfully JSON-RPC is very simple! 12 | # let's write some JSON to call "add" from 'mands/demo-python' with 2 parameters 13 | mkdir demo 14 | cd demo 15 | nano test_request.json 16 | 17 | 18 | { 19 | "service": "mands/demo-python", 20 | "req": { 21 | "method": "add", 22 | "params": [2, 2] 23 | } 24 | } 25 | 26 | 27 | # we can call this through a HTTP POST request to https://api.stackhut.com/run 28 | # let's use the "http" tool (https://github.com/jkbrzt/httpie) 29 | http POST https://api.stackhut.com/run @./test_request.json 30 | 31 | # the JSON-RPC response contains the return value in the "result" field 32 | # let's pipe it into "jq" (http://stedolan.github.io/jq/) 33 | http POST https://api.stackhut.com/run @./test_request.json | jq '.response.result' 34 | 35 | # so it turns out that 2 + 2 = 4, great! :) 36 | # and if we send invalid data... 37 | nano test_request.json 38 | http POST https://api.stackhut.com/run @./test_request.json 39 | 40 | # the JSON-RPC response now has an "error" field 41 | # the system caught the mistake and you can deal with it programmatically 42 | http POST https://api.stackhut.com/run @./test_request.json | jq '.response.error.code' 43 | 44 | # we hope this shows how you can call any StackHut service from your code 45 | # it's as easy as making a JSON-based POST request 46 | # thanks for your time - we can't wait to see how you make use of it... 47 | 48 | -------------------------------------------------------------------------------- /screencasts/use_script.md: -------------------------------------------------------------------------------- 1 | Tutorial - Using a Service In this screencast we quickly show how to use a StackHut service from within your application, be it written in Python, client-side/server-side JS, Ruby, .NET and more. 2 | 3 | 4 | 5 | Selecting a service You can find all manner of services, from video encoding, to complression, onine-compilation, web scraping, and more, on the StackHut repository (ref). Even better, develop or fork an exiting sservice. So, to start off with I'm going to use the ``demo-python`` service, which we show how to create in a corresponding screencast. So let's have a look here. First we have the interactive post playground, already configured with an example. So all StackHut services can be accessed and consumed via a direct HTTP POST request. On receving a request, the StackHut host platform will spin-up a container on demand to complete your request. To communicate with the service we have chosen to use JSON-RPC. As a result any StackHut service can easily be consumed by construuted a JSON-RPC request in any language - thankfully JSON-RPC is very simple and this is much simpler than it sounds! In this example JSON-RPC reqeust, we call the method ``helloName`` in the StackHut service ``example-python`` with a single parameter ``StackHut``. ``Params`` is a JSON list that contain any JSON types, i.e. flaots, strings, lists and objects. Let's run that and see what happens...wow, that was quick. So, internlly all we are doing is making a POST reuest to api.stackhut.com/run, the exat same code you would write yourself if acessessing StacHut from client-side JavaScript. Ok, now let's look at a successful request object. The ``response`` object is a JSON-RPC response, where the data we're after can be found in the ``result`` field - in this case a string containing the text ``Hello, StackHut :)``. Let's take a look at what other APIs this service allows...ok, so here is an add function, let's try that, but this time from the console. We can do this by using the great command-line tool ``http`` to sending the POST request, with content-type ``application/json`` to ``https://api.stackhut.com/run``. So here is a JSON-RPC request I made earlier --- and let's send it:: http POST https://api.stackhut.com/run < ./test_request.json So there's a bunch of output here, HTTP headers, and so on - but let's look at the HTTP response body - yep, here's the ``result`` object field. So it turns out 2 + 2 is 4, great! 6 | 7 | You can do this yourself from any language that support JSON HTTP requests, and hopefully you can see how simple it is to access a StacHut service within your own code. We also have client-side libraries to make this even easier under development - sign up for more information and please view other other screencasts on creating a StackHut service. 8 | 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.6.1 3 | commit = True 4 | tag = True 5 | tag_name = {new_version} 6 | message = Release {new_version} 7 | 8 | [bumpversion:file:setup.py] 9 | 10 | [bumpversion:file:stackhut_toolkit/__init__.py] 11 | 12 | [wheel] 13 | universal = 0 14 | 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import sys 4 | import os 5 | from setuptools import setup, find_packages 6 | 7 | def read(*paths): 8 | """Build a file path from *paths* and return the contents.""" 9 | with open(os.path.join(*paths), 'r') as f: 10 | return f.read() 11 | 12 | here = os.path.abspath(os.path.dirname(__file__)) 13 | 14 | # put package test requirements here 15 | requirements = [ 16 | "sh", 17 | "requests", 18 | "jinja2", 19 | "multipledispatch", 20 | "docker-py", 21 | "arrow", 22 | "PyYaml", 23 | "colorlog", 24 | "werkzeug", 25 | "json-rpc", 26 | "prompt_toolkit", 27 | "pygments", 28 | "stackhut-client >= 0.1.1", 29 | ] 30 | 31 | # put package test requirements here 32 | test_requirements = [] 33 | 34 | setup( 35 | name='stackhut-toolkit', 36 | version='0.6.1', 37 | description="Deploy classes as Microservices", 38 | long_description=read('readme_pip.rst'), 39 | license='Apache', 40 | author="StackHut", 41 | author_email='stackhut@stackhut.com', 42 | url='https://github.com/stackhut/stackhut-toolkit', 43 | # download_url = 'https://github.com/stackhut/stackhut-tool/tarball/0.1.0' 44 | packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", ""]), 45 | include_package_data=True, 46 | entry_points={ 47 | 'console_scripts': [ 48 | 'stackhut = stackhut_toolkit.__main__:main', 49 | ], 50 | }, 51 | install_requires=requirements, 52 | zip_safe=False, 53 | test_suite='tests', 54 | tests_require=test_requirements, 55 | keywords='stackhut', 56 | platforms=['POSIX'], 57 | classifiers=[ 58 | 'Development Status :: 2 - Pre-Alpha', 59 | 'Intended Audience :: Developers', 60 | 'Environment :: Console', 61 | 'Natural Language :: English', 62 | 'License :: OSI Approved :: Apache Software License', 63 | 'Operating System :: OS Independent', 64 | 'Programming Language :: Python', 65 | 'Programming Language :: Python :: 3', 66 | 'Programming Language :: Python :: 3.4', 67 | 'Topic :: Software Development', 68 | #'Private :: Do Not Upload', # hack to force invalid package for upload 69 | ], 70 | ) 71 | -------------------------------------------------------------------------------- /setup_freeze.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import sys 4 | import os 5 | from setuptools import find_packages 6 | from cx_Freeze import setup, Executable 7 | 8 | def read(*paths): 9 | """Build a file path from *paths* and return the contents.""" 10 | with open(os.path.join(*paths), 'r') as f: 11 | return f.read() 12 | 13 | here = os.path.abspath(os.path.dirname(__file__)) 14 | 15 | # put package test requirements here 16 | requirements = [ 17 | "wheel", 18 | "sh", 19 | "requests", 20 | "jinja2", 21 | "multipledispatch", 22 | "docker-py", 23 | "arrow", 24 | "PyYaml", 25 | "colorlog", 26 | "werkzeug", 27 | "json-rpc", 28 | "stackhut-client >= 0.1.1", 29 | ] 30 | 31 | # put package test requirements here 32 | test_requirements = [] 33 | 34 | options = { 35 | 'build_exe': { 36 | 'compressed': True, 37 | 'optimize': 2, 38 | 'include_files': ['stackhut_toolkit/res'] 39 | 40 | # 'init_script':'Console', 41 | # 'includes': [ 42 | # 'testfreeze_1', 43 | # 'testfreeze_2' 44 | # ], 45 | # 'path': sys.path + ['modules'] 46 | } 47 | } 48 | 49 | executables = [ 50 | Executable( 51 | # script='stackhut/__main__.py', 52 | script='stackhut.py', 53 | initScript='Console', 54 | ) 55 | 56 | ] 57 | 58 | 59 | setup( 60 | executables=executables, 61 | options=options, 62 | name='stackhut', 63 | version='0.5.8', 64 | description="Deploy classes as Microservices", 65 | long_description=read('readme_pip.rst'), 66 | license='Apache', 67 | author="StackHut", 68 | author_email='stackhut@stackhut.com', 69 | url='https://github.com/stackhut/stackhut-toolkit', 70 | # download_url = 'https://github.com/stackhut/stackhut-tool/tarball/0.1.0' 71 | packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", ""]), 72 | include_package_data=True, 73 | entry_points={ 74 | 'console_scripts': [ 75 | 'stackhut = stackhut_toolkit.__main__:main', 76 | ], 77 | }, 78 | install_requires=requirements, 79 | zip_safe=False, 80 | test_suite='tests', 81 | tests_require=test_requirements, 82 | keywords='stackhut', 83 | platforms=['POSIX'], 84 | classifiers=[ 85 | 'Development Status :: 2 - Pre-Alpha', 86 | 'Intended Audience :: Developers', 87 | 'Environment :: Console', 88 | 'Natural Language :: English', 89 | 'License :: OSI Approved :: Apache Software License', 90 | 'Operating System :: OS Independent', 91 | 'Programming Language :: Python', 92 | 'Programming Language :: Python :: 3', 93 | 'Programming Language :: Python :: 3.4', 94 | 'Topic :: Software Development', 95 | #'Private :: Do Not Upload', # hack to force invalid package for upload 96 | ], 97 | ) 98 | -------------------------------------------------------------------------------- /stackhut.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from stackhut_toolkit.__main__ import main 3 | 4 | if __name__ == '__main__': 5 | sys.exit(main()) 6 | 7 | -------------------------------------------------------------------------------- /stackhut_toolkit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # -*- coding: utf-8 -*- 16 | __version__ = '0.6.1' 17 | 18 | -------------------------------------------------------------------------------- /stackhut_toolkit/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2015 StackHut Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | import sys 17 | from . import __version__ 18 | from .common.commands import CmdRunner 19 | from .common import utils 20 | from .toolkit_utils import keen_client 21 | from .builder import get_docker 22 | 23 | from .commands import COMMANDS 24 | from .run_commands import COMMANDS as RUN_COMMANDS 25 | 26 | class ToolkitRunner(CmdRunner): 27 | def custom_error(self, e): 28 | import traceback 29 | # exception analytics 30 | try: 31 | dv = get_docker(_exit=False, verbose=False).client.version().get('Version') 32 | except: 33 | dv = None 34 | 35 | keen_client.send('cli_exception', 36 | dict(cmd=self.args.command, 37 | exception=repr(e), 38 | stackhut_version=__version__, 39 | docker_version=dv, 40 | os=sys.platform, 41 | python_version=sys.version, 42 | traceback=traceback.format_exc())) 43 | 44 | utils.log.info(":( If this reoccurs please open an issue at https://github.com/stackhut/stackhut " 45 | "or email toolkit@stackhut.com - thanks!") 46 | 47 | def custom_shutdown(self): 48 | keen_client.shutdown() 49 | 50 | def main(): 51 | runner = ToolkitRunner("StackHut Toolkit", __version__) 52 | # register the sub-commands 53 | runner.register_commands(COMMANDS + RUN_COMMANDS) 54 | # start 55 | retval = runner.start() 56 | return retval 57 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | common.barrister 4 | ~~~~~~~~~ 5 | 6 | A RPC toolkit for building lightweight reliable services. Ideal for 7 | both static and dynamic languages. 8 | 9 | http://common.barrister.bitmechanic.com/ 10 | 11 | :copyright: 2012 by James Cooper. 12 | :license: MIT, see LICENSE for more details. 13 | """ 14 | __version__ = '0.1.7.stackhut' 15 | 16 | from .runtime import contract_from_file, idgen_uuid, idgen_seq 17 | from .runtime import RpcException, Server, Filter, HttpTransport, InProcTransport 18 | from .runtime import Client, Batch 19 | from .runtime import Contract, Interface, Enum, Struct, Function 20 | from .runtime import err_response, ERR_PARSE, ERR_INVALID_REQ, ERR_METHOD_NOT_FOUND, \ 21 | ERR_INVALID_PARAMS, ERR_INTERNAL, ERR_UNKNOWN, ERR_INVALID_RESP 22 | from .parser import parse 23 | 24 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Actions.pxd: -------------------------------------------------------------------------------- 1 | 2 | cdef class Action: 3 | cdef perform(self, token_stream, text) 4 | cpdef same_as(self, other) 5 | 6 | cdef class Return(Action): 7 | cdef object value 8 | cdef perform(self, token_stream, text) 9 | cpdef same_as(self, other) 10 | 11 | cdef class Call(Action): 12 | cdef object function 13 | cdef perform(self, token_stream, text) 14 | cpdef same_as(self, other) 15 | 16 | cdef class Begin(Action): 17 | cdef object state_name 18 | cdef perform(self, token_stream, text) 19 | cpdef same_as(self, other) 20 | 21 | cdef class Ignore(Action): 22 | cdef perform(self, token_stream, text) 23 | 24 | cdef class Text(Action): 25 | cdef perform(self, token_stream, text) 26 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Actions.py: -------------------------------------------------------------------------------- 1 | #======================================================================= 2 | # 3 | # Python Lexical Analyser 4 | # 5 | # Actions for use in token specifications 6 | # 7 | #======================================================================= 8 | 9 | class Action(object): 10 | def perform(self, token_stream, text): 11 | pass # abstract 12 | 13 | def same_as(self, other): 14 | return self is other 15 | 16 | 17 | class Return(Action): 18 | """ 19 | Internal Plex action which causes |value| to 20 | be returned as the value of the associated token 21 | """ 22 | 23 | def __init__(self, value): 24 | self.value = value 25 | 26 | def perform(self, token_stream, text): 27 | return self.value 28 | 29 | def same_as(self, other): 30 | return isinstance(other, Return) and self.value == other.value 31 | 32 | def __repr__(self): 33 | return "Return(%s)" % repr(self.value) 34 | 35 | 36 | class Call(Action): 37 | """ 38 | Internal Plex action which causes a function to be called. 39 | """ 40 | 41 | def __init__(self, function): 42 | self.function = function 43 | 44 | def perform(self, token_stream, text): 45 | return self.function(token_stream, text) 46 | 47 | def __repr__(self): 48 | return "Call(%s)" % self.function.__name__ 49 | 50 | def same_as(self, other): 51 | return isinstance(other, Call) and self.function is other.function 52 | 53 | 54 | class Begin(Action): 55 | """ 56 | Begin(state_name) is a Plex action which causes the Scanner to 57 | enter the state |state_name|. See the docstring of Plex.Lexicon 58 | for more information. 59 | """ 60 | 61 | def __init__(self, state_name): 62 | self.state_name = state_name 63 | 64 | def perform(self, token_stream, text): 65 | token_stream.begin(self.state_name) 66 | 67 | def __repr__(self): 68 | return "Begin(%s)" % self.state_name 69 | 70 | def same_as(self, other): 71 | return isinstance(other, Begin) and self.state_name == other.state_name 72 | 73 | 74 | class Ignore(Action): 75 | """ 76 | IGNORE is a Plex action which causes its associated token 77 | to be ignored. See the docstring of Plex.Lexicon for more 78 | information. 79 | """ 80 | 81 | def perform(self, token_stream, text): 82 | return None 83 | 84 | def __repr__(self): 85 | return "IGNORE" 86 | 87 | 88 | IGNORE = Ignore() 89 | #IGNORE.__doc__ = Ignore.__doc__ 90 | 91 | 92 | class Text(Action): 93 | """ 94 | TEXT is a Plex action which causes the text of a token to 95 | be returned as the value of the token. See the docstring of 96 | Plex.Lexicon for more information. 97 | """ 98 | 99 | def perform(self, token_stream, text): 100 | return text 101 | 102 | def __repr__(self): 103 | return "TEXT" 104 | 105 | 106 | TEXT = Text() 107 | #TEXT.__doc__ = Text.__doc__ 108 | 109 | 110 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/DFA.py: -------------------------------------------------------------------------------- 1 | #======================================================================= 2 | # 3 | # Python Lexical Analyser 4 | # 5 | # Converting NFA to DFA 6 | # 7 | #======================================================================= 8 | 9 | 10 | 11 | from . import Machines 12 | from .Machines import LOWEST_PRIORITY 13 | from .Transitions import TransitionMap 14 | 15 | 16 | def nfa_to_dfa(old_machine, debug=None): 17 | """ 18 | Given a nondeterministic Machine, return a new equivalent 19 | Machine which is deterministic. 20 | """ 21 | # We build a new machine whose states correspond to sets of states 22 | # in the old machine. Initially we add a new state corresponding to 23 | # the epsilon-closure of each initial old state. Then we give transitions 24 | # to each new state which are the union of all transitions out of any 25 | # of the corresponding old states. The new state reached on a given 26 | # character is the one corresponding to the set of states reachable 27 | # on that character from any of the old states. As new combinations of 28 | # old states are created, new states are added as needed until closure 29 | # is reached. 30 | new_machine = Machines.FastMachine() 31 | state_map = StateMap(new_machine) 32 | # Seed the process using the initial states of the old machine. 33 | # Make the corresponding new states into initial states of the new 34 | # machine with the same names. 35 | for (key, old_state) in old_machine.initial_states.items(): 36 | new_state = state_map.old_to_new(epsilon_closure(old_state)) 37 | new_machine.make_initial_state(key, new_state) 38 | # Tricky bit here: we add things to the end of this list while we're 39 | # iterating over it. The iteration stops when closure is achieved. 40 | for new_state in new_machine.states: 41 | transitions = TransitionMap() 42 | for old_state in state_map.new_to_old(new_state): 43 | for event, old_target_states in old_state.transitions.items(): 44 | if event and old_target_states: 45 | transitions.add_set(event, set_epsilon_closure(old_target_states)) 46 | for event, old_states in transitions.items(): 47 | new_machine.add_transitions(new_state, event, state_map.old_to_new(old_states)) 48 | if debug: 49 | debug.write("\n===== State Mapping =====\n") 50 | state_map.dump(debug) 51 | return new_machine 52 | 53 | 54 | def set_epsilon_closure(state_set): 55 | """ 56 | Given a set of states, return the union of the epsilon 57 | closures of its member states. 58 | """ 59 | result = {} 60 | for state1 in state_set: 61 | for state2 in epsilon_closure(state1): 62 | result[state2] = 1 63 | return result 64 | 65 | 66 | def epsilon_closure(state): 67 | """ 68 | Return the set of states reachable from the given state 69 | by epsilon moves. 70 | """ 71 | # Cache the result 72 | result = state.epsilon_closure 73 | if result is None: 74 | result = {} 75 | state.epsilon_closure = result 76 | add_to_epsilon_closure(result, state) 77 | return result 78 | 79 | 80 | def add_to_epsilon_closure(state_set, state): 81 | """ 82 | Recursively add to |state_set| states reachable from the given state 83 | by epsilon moves. 84 | """ 85 | if not state_set.get(state, 0): 86 | state_set[state] = 1 87 | state_set_2 = state.transitions.get_epsilon() 88 | if state_set_2: 89 | for state2 in state_set_2: 90 | add_to_epsilon_closure(state_set, state2) 91 | 92 | 93 | class StateMap(object): 94 | """ 95 | Helper class used by nfa_to_dfa() to map back and forth between 96 | sets of states from the old machine and states of the new machine. 97 | """ 98 | new_machine = None # Machine 99 | old_to_new_dict = None # {(old_state,...) : new_state} 100 | new_to_old_dict = None # {id(new_state) : old_state_set} 101 | 102 | def __init__(self, new_machine): 103 | self.new_machine = new_machine 104 | self.old_to_new_dict = {} 105 | self.new_to_old_dict = {} 106 | 107 | def old_to_new(self, old_state_set): 108 | """ 109 | Return the state of the new machine corresponding to the 110 | set of old machine states represented by |state_set|. A new 111 | state will be created if necessary. If any of the old states 112 | are accepting states, the new state will be an accepting state 113 | with the highest priority action from the old states. 114 | """ 115 | key = self.make_key(old_state_set) 116 | new_state = self.old_to_new_dict.get(key, None) 117 | if not new_state: 118 | action = self.highest_priority_action(old_state_set) 119 | new_state = self.new_machine.new_state(action) 120 | self.old_to_new_dict[key] = new_state 121 | self.new_to_old_dict[id(new_state)] = old_state_set 122 | #for old_state in old_state_set.keys(): 123 | #new_state.merge_actions(old_state) 124 | return new_state 125 | 126 | def highest_priority_action(self, state_set): 127 | best_action = None 128 | best_priority = LOWEST_PRIORITY 129 | for state in state_set: 130 | priority = state.action_priority 131 | if priority > best_priority: 132 | best_action = state.action 133 | best_priority = priority 134 | return best_action 135 | 136 | # def old_to_new_set(self, old_state_set): 137 | # """ 138 | # Return the new state corresponding to a set of old states as 139 | # a singleton set. 140 | # """ 141 | # return {self.old_to_new(old_state_set):1} 142 | 143 | def new_to_old(self, new_state): 144 | """Given a new state, return a set of corresponding old states.""" 145 | return self.new_to_old_dict[id(new_state)] 146 | 147 | def make_key(self, state_set): 148 | """ 149 | Convert a set of states into a uniquified 150 | sorted tuple suitable for use as a dictionary key. 151 | """ 152 | lst = list(state_set) 153 | lst.sort() 154 | return tuple(lst) 155 | 156 | def dump(self, file): 157 | from .Transitions import state_set_str 158 | 159 | for new_state in self.new_machine.states: 160 | old_state_set = self.new_to_old_dict[id(new_state)] 161 | file.write(" State %s <-- %s\n" % ( 162 | new_state['number'], state_set_str(old_state_set))) 163 | 164 | 165 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Errors.py: -------------------------------------------------------------------------------- 1 | #======================================================================= 2 | # 3 | # Python Lexical Analyser 4 | # 5 | # Exception classes 6 | # 7 | #======================================================================= 8 | 9 | 10 | class PlexError(Exception): 11 | message = "" 12 | 13 | 14 | class PlexTypeError(PlexError, TypeError): 15 | pass 16 | 17 | 18 | class PlexValueError(PlexError, ValueError): 19 | pass 20 | 21 | 22 | class InvalidRegex(PlexError): 23 | pass 24 | 25 | 26 | class InvalidToken(PlexError): 27 | def __init__(self, token_number, message): 28 | PlexError.__init__(self, "Token number %d: %s" % (token_number, message)) 29 | 30 | 31 | class InvalidScanner(PlexError): 32 | pass 33 | 34 | 35 | class AmbiguousAction(PlexError): 36 | message = "Two tokens with different actions can match the same string" 37 | 38 | def __init__(self): 39 | pass 40 | 41 | 42 | class UnrecognizedInput(PlexError): 43 | scanner = None 44 | position = None 45 | state_name = None 46 | 47 | def __init__(self, scanner, state_name): 48 | self.scanner = scanner 49 | self.position = scanner.get_position() 50 | self.state_name = state_name 51 | 52 | def __str__(self): 53 | return ("'%s', line %d, char %d: Token not recognised in state %r" % ( 54 | self.position + (self.state_name,))) 55 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Lexicons.py: -------------------------------------------------------------------------------- 1 | #======================================================================= 2 | # 3 | # Python Lexical Analyser 4 | # 5 | # Lexical Analyser Specification 6 | # 7 | #======================================================================= 8 | 9 | 10 | 11 | import types 12 | 13 | from . import Actions 14 | from . import DFA 15 | from . import Errors 16 | from . import Machines 17 | from . import Regexps 18 | 19 | # debug_flags for Lexicon constructor 20 | DUMP_NFA = 1 21 | DUMP_DFA = 2 22 | 23 | 24 | class State(object): 25 | """ 26 | This class is used as part of a Plex.Lexicon specification to 27 | introduce a user-defined state. 28 | 29 | Constructor: 30 | 31 | State(name, token_specifications) 32 | """ 33 | 34 | name = None 35 | tokens = None 36 | 37 | def __init__(self, name, tokens): 38 | self.name = name 39 | self.tokens = tokens 40 | 41 | 42 | class Lexicon(object): 43 | """ 44 | Lexicon(specification) builds a lexical analyser from the given 45 | |specification|. The specification consists of a list of 46 | specification items. Each specification item may be either: 47 | 48 | 1) A token definition, which is a tuple: 49 | 50 | (pattern, action) 51 | 52 | The |pattern| is a regular axpression built using the 53 | constructors defined in the Plex module. 54 | 55 | The |action| is the action to be performed when this pattern 56 | is recognised (see below). 57 | 58 | 2) A state definition: 59 | 60 | State(name, tokens) 61 | 62 | where |name| is a character string naming the state, 63 | and |tokens| is a list of token definitions as 64 | above. The meaning and usage of states is described 65 | below. 66 | 67 | Actions 68 | ------- 69 | 70 | The |action| in a token specication may be one of three things: 71 | 72 | 1) A function, which is called as follows: 73 | 74 | function(scanner, text) 75 | 76 | where |scanner| is the relevant Scanner instance, and |text| 77 | is the matched text. If the function returns anything 78 | other than None, that value is returned as the value of the 79 | token. If it returns None, scanning continues as if the IGNORE 80 | action were specified (see below). 81 | 82 | 2) One of the following special actions: 83 | 84 | IGNORE means that the recognised characters will be treated as 85 | white space and ignored. Scanning will continue until 86 | the next non-ignored token is recognised before returning. 87 | 88 | TEXT causes the scanned text itself to be returned as the 89 | value of the token. 90 | 91 | 3) Any other value, which is returned as the value of the token. 92 | 93 | States 94 | ------ 95 | 96 | At any given time, the scanner is in one of a number of states. 97 | Associated with each state is a set of possible tokens. When scanning, 98 | only tokens associated with the current state are recognised. 99 | 100 | There is a default state, whose name is the empty string. Token 101 | definitions which are not inside any State definition belong to 102 | the default state. 103 | 104 | The initial state of the scanner is the default state. The state can 105 | be changed in one of two ways: 106 | 107 | 1) Using Begin(state_name) as the action of a token. 108 | 109 | 2) Calling the begin(state_name) method of the Scanner. 110 | 111 | To change back to the default state, use '' as the state name. 112 | """ 113 | 114 | machine = None # Machine 115 | tables = None # StateTableMachine 116 | 117 | def __init__(self, specifications, debug=None, debug_flags=7, timings=None): 118 | if type(specifications) != list: 119 | raise Errors.InvalidScanner("Scanner definition is not a list") 120 | if timings: 121 | from .Timing import time 122 | 123 | total_time = 0.0 124 | time1 = time() 125 | nfa = Machines.Machine() 126 | default_initial_state = nfa.new_initial_state('') 127 | token_number = 1 128 | for spec in specifications: 129 | if isinstance(spec, State): 130 | user_initial_state = nfa.new_initial_state(spec.name) 131 | for token in spec.tokens: 132 | self.add_token_to_machine( 133 | nfa, user_initial_state, token, token_number) 134 | token_number += 1 135 | elif type(spec) == tuple: 136 | self.add_token_to_machine( 137 | nfa, default_initial_state, spec, token_number) 138 | token_number += 1 139 | else: 140 | raise Errors.InvalidToken( 141 | token_number, 142 | "Expected a token definition (tuple) or State instance") 143 | if timings: 144 | time2 = time() 145 | total_time = total_time + (time2 - time1) 146 | time3 = time() 147 | if debug and (debug_flags & 1): 148 | debug.write("\n============= NFA ===========\n") 149 | nfa.dump(debug) 150 | dfa = DFA.nfa_to_dfa(nfa, debug=(debug_flags & 3) == 3 and debug) 151 | if timings: 152 | time4 = time() 153 | total_time = total_time + (time4 - time3) 154 | if debug and (debug_flags & 2): 155 | debug.write("\n============= DFA ===========\n") 156 | dfa.dump(debug) 157 | if timings: 158 | timings.write("Constructing NFA : %5.2f\n" % (time2 - time1)) 159 | timings.write("Converting to DFA: %5.2f\n" % (time4 - time3)) 160 | timings.write("TOTAL : %5.2f\n" % total_time) 161 | self.machine = dfa 162 | 163 | def add_token_to_machine(self, machine, initial_state, token_spec, token_number): 164 | try: 165 | (re, action_spec) = self.parse_token_definition(token_spec) 166 | # Disabled this -- matching empty strings can be useful 167 | #if re.nullable: 168 | # raise Errors.InvalidToken( 169 | # token_number, "Pattern can match 0 input symbols") 170 | if isinstance(action_spec, Actions.Action): 171 | action = action_spec 172 | else: 173 | try: 174 | action_spec.__call__ 175 | except AttributeError: 176 | action = Actions.Return(action_spec) 177 | else: 178 | action = Actions.Call(action_spec) 179 | final_state = machine.new_state() 180 | re.build_machine(machine, initial_state, final_state, 181 | match_bol=1, nocase=0) 182 | final_state.set_action(action, priority=-token_number) 183 | except Errors.PlexError as e: 184 | raise e.__class__("Token number %d: %s" % (token_number, e)) 185 | 186 | def parse_token_definition(self, token_spec): 187 | if type(token_spec) != tuple: 188 | raise Errors.InvalidToken("Token definition is not a tuple") 189 | if len(token_spec) != 2: 190 | raise Errors.InvalidToken("Wrong number of items in token definition") 191 | pattern, action = token_spec 192 | if not isinstance(pattern, Regexps.RE): 193 | raise Errors.InvalidToken("Pattern is not an RE instance") 194 | return (pattern, action) 195 | 196 | def get_initial_state(self, name): 197 | return self.machine.get_initial_state(name) 198 | 199 | 200 | 201 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Scanners.pxd: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import cython 4 | 5 | from Cython.Plex.Actions cimport Action 6 | 7 | cdef class Scanner: 8 | 9 | cdef public lexicon 10 | cdef public stream 11 | cdef public name 12 | cdef public unicode buffer 13 | cdef public Py_ssize_t buf_start_pos 14 | cdef public Py_ssize_t next_pos 15 | cdef public Py_ssize_t cur_pos 16 | cdef public Py_ssize_t cur_line 17 | cdef public Py_ssize_t cur_line_start 18 | cdef public Py_ssize_t start_pos 19 | cdef public Py_ssize_t start_line 20 | cdef public Py_ssize_t start_col 21 | cdef public text 22 | cdef public initial_state # int? 23 | cdef public state_name 24 | cdef public list queue 25 | cdef public bint trace 26 | cdef public cur_char 27 | cdef public long input_state 28 | 29 | cdef public level 30 | 31 | @cython.locals(input_state=long) 32 | cdef next_char(self) 33 | @cython.locals(action=Action) 34 | cpdef tuple read(self) 35 | cdef tuple scan_a_token(self) 36 | cdef tuple position(self) 37 | 38 | @cython.locals(cur_pos=long, cur_line=long, cur_line_start=long, 39 | input_state=long, next_pos=long, state=dict, 40 | buf_start_pos=long, buf_len=long, buf_index=long, 41 | trace=bint, discard=long, data=unicode, buffer=unicode) 42 | cdef run_machine_inlined(self) 43 | 44 | cdef begin(self, state) 45 | cdef produce(self, value, text = *) 46 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Timing.py: -------------------------------------------------------------------------------- 1 | # 2 | # Get time in platform-dependent way 3 | # 4 | 5 | 6 | 7 | import os 8 | from sys import platform, exit, stderr 9 | 10 | if platform == 'mac': 11 | import MacOS 12 | def time(): 13 | return MacOS.GetTicks() / 60.0 14 | timekind = "real" 15 | elif hasattr(os, 'times'): 16 | def time(): 17 | t = os.times() 18 | return t[0] + t[1] 19 | timekind = "cpu" 20 | else: 21 | stderr.write( 22 | "Don't know how to get time on platform %s\n" % repr(platform)) 23 | exit(1) 24 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Traditional.py: -------------------------------------------------------------------------------- 1 | #======================================================================= 2 | # 3 | # Python Lexical Analyser 4 | # 5 | # Traditional Regular Expression Syntax 6 | # 7 | #======================================================================= 8 | 9 | 10 | 11 | from .Regexps import Alt, Seq, Rep, Rep1, Opt, Any, AnyBut, Bol, Eol, Char 12 | from .Errors import PlexError 13 | 14 | 15 | class RegexpSyntaxError(PlexError): 16 | pass 17 | 18 | 19 | def re(s): 20 | """ 21 | Convert traditional string representation of regular expression |s| 22 | into Plex representation. 23 | """ 24 | return REParser(s).parse_re() 25 | 26 | 27 | class REParser(object): 28 | def __init__(self, s): 29 | self.s = s 30 | self.i = -1 31 | self.end = 0 32 | next(self) 33 | 34 | def parse_re(self): 35 | re = self.parse_alt() 36 | if not self.end: 37 | self.error("Unexpected %s" % repr(self.c)) 38 | return re 39 | 40 | def parse_alt(self): 41 | """Parse a set of alternative regexps.""" 42 | re = self.parse_seq() 43 | if self.c == '|': 44 | re_list = [re] 45 | while self.c == '|': 46 | next(self) 47 | re_list.append(self.parse_seq()) 48 | re = Alt(*re_list) 49 | return re 50 | 51 | def parse_seq(self): 52 | """Parse a sequence of regexps.""" 53 | re_list = [] 54 | while not self.end and not self.c in "|)": 55 | re_list.append(self.parse_mod()) 56 | return Seq(*re_list) 57 | 58 | def parse_mod(self): 59 | """Parse a primitive regexp followed by *, +, ? modifiers.""" 60 | re = self.parse_prim() 61 | while not self.end and self.c in "*+?": 62 | if self.c == '*': 63 | re = Rep(re) 64 | elif self.c == '+': 65 | re = Rep1(re) 66 | else: # self.c == '?' 67 | re = Opt(re) 68 | next(self) 69 | return re 70 | 71 | def parse_prim(self): 72 | """Parse a primitive regexp.""" 73 | c = self.get() 74 | if c == '.': 75 | re = AnyBut("\n") 76 | elif c == '^': 77 | re = Bol 78 | elif c == '$': 79 | re = Eol 80 | elif c == '(': 81 | re = self.parse_alt() 82 | self.expect(')') 83 | elif c == '[': 84 | re = self.parse_charset() 85 | self.expect(']') 86 | else: 87 | if c == '\\': 88 | c = self.get() 89 | re = Char(c) 90 | return re 91 | 92 | def parse_charset(self): 93 | """Parse a charset. Does not include the surrounding [].""" 94 | char_list = [] 95 | invert = 0 96 | if self.c == '^': 97 | invert = 1 98 | next(self) 99 | if self.c == ']': 100 | char_list.append(']') 101 | next(self) 102 | while not self.end and self.c != ']': 103 | c1 = self.get() 104 | if self.c == '-' and self.lookahead(1) != ']': 105 | next(self) 106 | c2 = self.get() 107 | for a in range(ord(c1), ord(c2) + 1): 108 | char_list.append(chr(a)) 109 | else: 110 | char_list.append(c1) 111 | chars = ''.join(char_list) 112 | if invert: 113 | return AnyBut(chars) 114 | else: 115 | return Any(chars) 116 | 117 | def __next__(self): 118 | """Advance to the next char.""" 119 | s = self.s 120 | i = self.i = self.i + 1 121 | if i < len(s): 122 | self.c = s[i] 123 | else: 124 | self.c = '' 125 | self.end = 1 126 | 127 | def get(self): 128 | if self.end: 129 | self.error("Premature end of string") 130 | c = self.c 131 | next(self) 132 | return c 133 | 134 | def lookahead(self, n): 135 | """Look ahead n chars.""" 136 | j = self.i + n 137 | if j < len(self.s): 138 | return self.s[j] 139 | else: 140 | return '' 141 | 142 | def expect(self, c): 143 | """ 144 | Expect to find character |c| at current position. 145 | Raises an exception otherwise. 146 | """ 147 | if self.c == c: 148 | next(self) 149 | else: 150 | self.error("Missing %s" % repr(c)) 151 | 152 | def error(self, mess): 153 | """Raise exception to signal syntax error in regexp.""" 154 | raise RegexpSyntaxError("Syntax error in regexp %s at position %d: %s" % ( 155 | repr(self.s), self.i, mess)) 156 | 157 | 158 | 159 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/Transitions.py: -------------------------------------------------------------------------------- 1 | # 2 | # Plex - Transition Maps 3 | # 4 | # This version represents state sets directly as dicts for speed. 5 | # 6 | 7 | 8 | 9 | from sys import maxsize as maxint 10 | 11 | 12 | class TransitionMap(object): 13 | """ 14 | A TransitionMap maps an input event to a set of states. 15 | An input event is one of: a range of character codes, 16 | the empty string (representing an epsilon move), or one 17 | of the special symbols BOL, EOL, EOF. 18 | 19 | For characters, this implementation compactly represents 20 | the map by means of a list: 21 | 22 | [code_0, states_0, code_1, states_1, code_2, states_2, 23 | ..., code_n-1, states_n-1, code_n] 24 | 25 | where |code_i| is a character code, and |states_i| is a 26 | set of states corresponding to characters with codes |c| 27 | in the range |code_i| <= |c| <= |code_i+1|. 28 | 29 | The following invariants hold: 30 | n >= 1 31 | code_0 == -maxint 32 | code_n == maxint 33 | code_i < code_i+1 for i in 0..n-1 34 | states_0 == states_n-1 35 | 36 | Mappings for the special events '', BOL, EOL, EOF are 37 | kept separately in a dictionary. 38 | """ 39 | 40 | map = None # The list of codes and states 41 | special = None # Mapping for special events 42 | 43 | def __init__(self, map=None, special=None): 44 | if not map: 45 | map = [-maxint, {}, maxint] 46 | if not special: 47 | special = {} 48 | self.map = map 49 | self.special = special 50 | #self.check() ### 51 | 52 | def add(self, event, new_state, 53 | TupleType=tuple): 54 | """ 55 | Add transition to |new_state| on |event|. 56 | """ 57 | if type(event) is TupleType: 58 | code0, code1 = event 59 | i = self.split(code0) 60 | j = self.split(code1) 61 | map = self.map 62 | while i < j: 63 | map[i + 1][new_state] = 1 64 | i += 2 65 | else: 66 | self.get_special(event)[new_state] = 1 67 | 68 | def add_set(self, event, new_set, 69 | TupleType=tuple): 70 | """ 71 | Add transitions to the states in |new_set| on |event|. 72 | """ 73 | if type(event) is TupleType: 74 | code0, code1 = event 75 | i = self.split(code0) 76 | j = self.split(code1) 77 | map = self.map 78 | while i < j: 79 | map[i + 1].update(new_set) 80 | i += 2 81 | else: 82 | self.get_special(event).update(new_set) 83 | 84 | def get_epsilon(self, 85 | none=None): 86 | """ 87 | Return the mapping for epsilon, or None. 88 | """ 89 | return self.special.get('', none) 90 | 91 | def iteritems(self, 92 | len=len): 93 | """ 94 | Return the mapping as an iterable of ((code1, code2), state_set) and 95 | (special_event, state_set) pairs. 96 | """ 97 | result = [] 98 | map = self.map 99 | else_set = map[1] 100 | i = 0 101 | n = len(map) - 1 102 | code0 = map[0] 103 | while i < n: 104 | set = map[i + 1] 105 | code1 = map[i + 2] 106 | if set or else_set: 107 | result.append(((code0, code1), set)) 108 | code0 = code1 109 | i += 2 110 | for event, set in self.special.items(): 111 | if set: 112 | result.append((event, set)) 113 | return iter(result) 114 | 115 | items = iteritems 116 | 117 | # ------------------- Private methods -------------------- 118 | 119 | def split(self, code, 120 | len=len, maxint=maxint): 121 | """ 122 | Search the list for the position of the split point for |code|, 123 | inserting a new split point if necessary. Returns index |i| such 124 | that |code| == |map[i]|. 125 | """ 126 | # We use a funky variation on binary search. 127 | map = self.map 128 | hi = len(map) - 1 129 | # Special case: code == map[-1] 130 | if code == maxint: 131 | return hi 132 | # General case 133 | lo = 0 134 | # loop invariant: map[lo] <= code < map[hi] and hi - lo >= 2 135 | while hi - lo >= 4: 136 | # Find midpoint truncated to even index 137 | mid = ((lo + hi) // 2) & ~1 138 | if code < map[mid]: 139 | hi = mid 140 | else: 141 | lo = mid 142 | # map[lo] <= code < map[hi] and hi - lo == 2 143 | if map[lo] == code: 144 | return lo 145 | else: 146 | map[hi:hi] = [code, map[hi - 1].copy()] 147 | #self.check() ### 148 | return hi 149 | 150 | def get_special(self, event): 151 | """ 152 | Get state set for special event, adding a new entry if necessary. 153 | """ 154 | special = self.special 155 | set = special.get(event, None) 156 | if not set: 157 | set = {} 158 | special[event] = set 159 | return set 160 | 161 | # --------------------- Conversion methods ----------------------- 162 | 163 | def __str__(self): 164 | map_strs = [] 165 | map = self.map 166 | n = len(map) 167 | i = 0 168 | while i < n: 169 | code = map[i] 170 | if code == -maxint: 171 | code_str = "-inf" 172 | elif code == maxint: 173 | code_str = "inf" 174 | else: 175 | code_str = str(code) 176 | map_strs.append(code_str) 177 | i += 1 178 | if i < n: 179 | map_strs.append(state_set_str(map[i])) 180 | i += 1 181 | special_strs = {} 182 | for event, set in self.special.items(): 183 | special_strs[event] = state_set_str(set) 184 | return "[%s]+%s" % ( 185 | ','.join(map_strs), 186 | special_strs 187 | ) 188 | 189 | # --------------------- Debugging methods ----------------------- 190 | 191 | def check(self): 192 | """Check data structure integrity.""" 193 | if not self.map[-3] < self.map[-1]: 194 | print(self) 195 | assert 0 196 | 197 | def dump(self, file): 198 | map = self.map 199 | i = 0 200 | n = len(map) - 1 201 | while i < n: 202 | self.dump_range(map[i], map[i + 2], map[i + 1], file) 203 | i += 2 204 | for event, set in self.special.items(): 205 | if set: 206 | if not event: 207 | event = 'empty' 208 | self.dump_trans(event, set, file) 209 | 210 | def dump_range(self, code0, code1, set, file): 211 | if set: 212 | if code0 == -maxint: 213 | if code1 == maxint: 214 | k = "any" 215 | else: 216 | k = "< %s" % self.dump_char(code1) 217 | elif code1 == maxint: 218 | k = "> %s" % self.dump_char(code0 - 1) 219 | elif code0 == code1 - 1: 220 | k = self.dump_char(code0) 221 | else: 222 | k = "%s..%s" % (self.dump_char(code0), 223 | self.dump_char(code1 - 1)) 224 | self.dump_trans(k, set, file) 225 | 226 | def dump_char(self, code): 227 | if 0 <= code <= 255: 228 | return repr(chr(code)) 229 | else: 230 | return "chr(%d)" % code 231 | 232 | def dump_trans(self, key, set, file): 233 | file.write(" %s --> %s\n" % (key, self.dump_set(set))) 234 | 235 | def dump_set(self, set): 236 | return state_set_str(set) 237 | 238 | 239 | # 240 | # State set manipulation functions 241 | # 242 | 243 | #def merge_state_sets(set1, set2): 244 | # for state in set2.keys(): 245 | # set1[state] = 1 246 | 247 | def state_set_str(set): 248 | return "[%s]" % ','.join(["S%d" % state.number for state in set]) 249 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/cythonplex3/__init__.py: -------------------------------------------------------------------------------- 1 | #======================================================================= 2 | # 3 | # Python Lexical Analyser 4 | # 5 | #======================================================================= 6 | 7 | """ 8 | The Plex module provides lexical analysers with similar capabilities 9 | to GNU Flex. The following classes and functions are exported; 10 | see the attached docstrings for more information. 11 | 12 | Scanner For scanning a character stream under the 13 | direction of a Lexicon. 14 | 15 | Lexicon For constructing a lexical definition 16 | to be used by a Scanner. 17 | 18 | Str, Any, AnyBut, AnyChar, Seq, Alt, Opt, Rep, Rep1, 19 | Bol, Eol, Eof, Empty 20 | 21 | Regular expression constructors, for building pattern 22 | definitions for a Lexicon. 23 | 24 | State For defining scanner states when creating a 25 | Lexicon. 26 | 27 | TEXT, IGNORE, Begin 28 | 29 | Actions for associating with patterns when 30 | creating a Lexicon. 31 | """ 32 | 33 | 34 | 35 | from .Actions import TEXT, IGNORE, Begin 36 | from .Lexicons import Lexicon, State 37 | from .Regexps import RE, Seq, Alt, Rep1, Empty, Str, Any, AnyBut, AnyChar, Range 38 | from .Regexps import Opt, Rep, Bol, Eol, Eof, Case, NoCase 39 | from .Scanners import Scanner 40 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/barrister/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class ConfigError(Exception): 3 | """ 4 | Parent class for errors caused by bad configuration. 5 | """ 6 | def __init__(self, message, *args, **kwargs): 7 | self.message = message 8 | 9 | def __str__(self): 10 | return '%s: %s' % (self.__class__.__name__, self.message) 11 | 12 | 13 | class InvalidFunctionError(ConfigError): 14 | """ 15 | Error raised when a function definition is not valid. 16 | """ 17 | pass 18 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/commands.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Module performs basic command handling infrastructure for subcommands 16 | """ 17 | import os 18 | import abc 19 | import argparse 20 | from . import utils 21 | from .config import HutfileCfg 22 | 23 | 24 | class CmdRunner: 25 | def __init__(self, title, version): 26 | self.title = title 27 | self.args = None 28 | # Parse the cmd args 29 | self.parser = argparse.ArgumentParser(description=title) 30 | self.parser.add_argument('-V', '--version', help="{} Version".format(title), 31 | action="version", version="%(prog)s {}".format(version)) 32 | self.parser.add_argument('-v', dest='verbose', help="Verbose mode", action='store_true') 33 | self.parser.add_argument('-s', dest='server', help=argparse.SUPPRESS) 34 | 35 | def register_commands(self, cmds): 36 | metavar = '{{{}}}'.format(str.join(',', [cmd.name for cmd in cmds if cmd.visible])) 37 | subparsers = self.parser.add_subparsers(title="{} Commands".format(self.title), dest='command', metavar=metavar) 38 | 39 | for cmd in cmds: 40 | if cmd.visible: 41 | sp = subparsers.add_parser(cmd.name, help=cmd.description, description=cmd.description) 42 | else: 43 | sp = subparsers.add_parser(cmd.name) 44 | 45 | sp.set_defaults(func=cmd) 46 | cmd.register(sp) 47 | 48 | def custom_error(self, e): 49 | pass 50 | 51 | def custom_shutdown(self): 52 | pass 53 | 54 | def start(self): 55 | # parse the args 56 | self.args = self.parser.parse_args() 57 | if self.args.command is None: 58 | self.parser.print_help() 59 | self.parser.exit(0, "No command given\n") 60 | 61 | # General App Setup 62 | if self.args.server: 63 | utils.SERVER_URL = self.args.server 64 | 65 | utils.setup_logging(self.args.verbose) 66 | utils.log.info("Starting {}".format(self.title)) 67 | 68 | try: 69 | # dispatch to correct cmd class - i.e. build, compile, run, etc. 70 | subfunc = self.args.func(self.args) 71 | retval = subfunc.run() 72 | except AssertionError as e: 73 | # if len(e.args) > 0: 74 | # [utils.log.error(x) for x in e.args] 75 | utils.log.error(str(e)) 76 | return 1 77 | 78 | except Exception as e: 79 | # general / unhandled exceptions here 80 | 81 | # if len(e.args) > 0: 82 | # [utils.log.error(x) for x in e.args] 83 | utils.log.error("{}: {}".format(type(e).__name__, str(e))) 84 | 85 | self.custom_error(e) 86 | 87 | if self.args.verbose: 88 | raise e 89 | else: 90 | utils.log.info("Exiting, run in verbose mode (stackhut -v ...) for more information") 91 | return 1 92 | 93 | finally: 94 | self.custom_shutdown() 95 | 96 | # all done 97 | return retval if retval else 0 98 | 99 | 100 | ################################################################################################### 101 | # StackHut Commands Handling 102 | class BaseCmd: 103 | """The Base Command implementing common func""" 104 | visible = True 105 | name = '' 106 | description = "" 107 | 108 | @staticmethod 109 | def register(sp): 110 | pass 111 | 112 | def __init__(self, args): 113 | self.args = args 114 | 115 | @abc.abstractmethod 116 | def run(self): 117 | """Main entry point for a command with parsed cmd args""" 118 | pass 119 | 120 | class HutCmd(BaseCmd): 121 | """Hut Commands are run from a Hut stack dir requiring a Hutfile""" 122 | def __init__(self, args): 123 | super().__init__(args) 124 | # import the hutfile 125 | self.hutcfg = HutfileCfg() 126 | # create stackhut dir if not present 127 | # os.mkdir(utils.STACKHUT_DIR) if not os.path.exists(utils.STACKHUT_DIR) else None 128 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Config files used by the platform 16 | """ 17 | import os 18 | import re 19 | import stat 20 | import uuid 21 | import json 22 | import yaml 23 | from .utils import log 24 | 25 | class UserCfg(dict): 26 | """ 27 | UserConfig configuration handling 28 | Wrapper class around dict that uses a json backing store 29 | """ 30 | show_keys = ['username', 'hash', 'send_analytics'] 31 | keep_keys = ['send_analytics', 'm_id'] 32 | config_version = 2 33 | anon_username = 'anonymous' 34 | config_fpath = os.path.expanduser(os.path.join('~', '.stackhut.cfg')) 35 | 36 | def __init__(self): 37 | super().__init__() 38 | if os.path.exists(self.config_fpath): 39 | with open(self.config_fpath, 'r') as f: 40 | self.update(json.load(f)) 41 | if self.get('config_version', 0) < self.config_version: 42 | self.wipe() 43 | raise AssertionError("Resetting after config file version update, please rerun command") 44 | else: 45 | # create with correct file permissions 46 | open(self.config_fpath, 'w').close() 47 | os.chmod(self.config_fpath, stat.S_IRUSR | stat.S_IWUSR) 48 | self.wipe() 49 | 50 | self.ask_analytics() 51 | 52 | def ask_analytics(self): 53 | def agree(): 54 | while True: 55 | x = input("Agree to send analytics [Y/N]: ").capitalize() 56 | if x.startswith('Y'): 57 | return True 58 | if x.startswith('N'): 59 | return False 60 | 61 | if self.get('send_analytics') is None: 62 | log.info("Welcome to StackHut - thank you for installing the Toolkit") 63 | log.info("To help us improve StackHut we'd like to send some usage and error data for analytics") 64 | log.info("We'd really like it if you could help us with this, however if you'd like to opt out please enter 'N'") 65 | self['send_analytics'] = agree() 66 | self['m_id'] = str(uuid.uuid4()) 67 | self.save() 68 | log.info("Thanks, your choice has been saved.") 69 | 70 | def save(self): 71 | with open(self.config_fpath, 'w') as f: 72 | json.dump(self, f, indent=4) 73 | 74 | def wipe(self): 75 | """blank out the cfg file""" 76 | x = {k: self.get(k) for k in self.keep_keys} 77 | self.clear() 78 | self.update(x) 79 | self['config_version'] = self.config_version 80 | self['username'] = self.anon_username 81 | self['hash'] = None 82 | self.save() 83 | 84 | def assert_valid_user(self): 85 | """Make sure user has valid account to deploy""" 86 | if self.username == self.anon_username: 87 | raise AssertionError("Please login first with 'stackhut login' using your credentials from the StackHut website") 88 | 89 | @property 90 | def username(self): 91 | return self['username'] 92 | 93 | @property 94 | def send_analytics(self): 95 | return self['send_analytics'] 96 | 97 | @property 98 | def analytics_ids(self): 99 | # if ('send_analytics' not in self) or (self.logged_in and 'u_id' not in self): 100 | # raise AssertionError("Config file error - please delete {} and try again".format(CFGFILE)) 101 | if self.send_analytics: 102 | return dict(m_id=self['m_id'], u_id=self.get('u_id')) 103 | else: 104 | return None 105 | 106 | class HutfileCfg: 107 | re_check_name = re.compile('^[a-z0-9-_]+$') 108 | 109 | """Hutfile configuration file handling""" 110 | def __init__(self): 111 | # import the hutfile 112 | if os.path.exists('Hutfile.yaml'): 113 | hutfile_fname = 'Hutfile.yaml' 114 | elif os.path.exists('Hutfile'): 115 | hutfile_fname = 'Hutfile' 116 | else: 117 | raise AssertionError("Cannot find 'Hutfile.yaml', is this a StackHut project dir?") 118 | 119 | with open(hutfile_fname, 'r') as f: 120 | hutfile = yaml.safe_load(f) 121 | 122 | # TODO - validation 123 | # get vals from the hutfile 124 | self.name = hutfile['name'] 125 | self.assert_valid_name(self.name) 126 | self.version = hutfile.get('version', 'latest') 127 | 128 | # self.email = hutfile['contact'] 129 | self.description = hutfile['description'] 130 | self.github_url = hutfile.get('github_url', None) 131 | 132 | # copy files and dirs separetly 133 | files = hutfile.get('files', []) 134 | self.files = [f for f in files if os.path.isfile(f)] 135 | self.dirs = [d for d in files if os.path.isdir(d)] 136 | 137 | self.persistent = hutfile.get('persistent', True) 138 | self.private = hutfile.get('private', False) 139 | 140 | self.os_deps = hutfile.get('os_deps', []) 141 | self.docker_cmds = hutfile.get('docker_cmds', []) 142 | self.baseos = hutfile['baseos'] 143 | self.stack = hutfile['stack'] 144 | 145 | @staticmethod 146 | def assert_valid_name(name): 147 | if HutfileCfg.re_check_name.match(name) is None: 148 | raise AssertionError("'{}' is not a valid service name, must be [a-z0-9-_]".format(name)) 149 | 150 | @property 151 | def from_image(self): 152 | return "{}-{}".format(self.baseos, self.stack) 153 | 154 | def service_short_name(self, username): 155 | """Returns the StackHut service name for the image""" 156 | return "{}/{}:{}".format(username, self.name, self.version) 157 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/exceptions.py: -------------------------------------------------------------------------------- 1 | # expose the barrister exceptions directly until other exceptions are needed 2 | from .barrister.exceptions import ConfigError # NOQA 3 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/runtime/backends.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | StackHut IO Handling on local and cloud backends 16 | """ 17 | import abc 18 | import os 19 | import json 20 | import shutil 21 | import threading 22 | from queue import Queue 23 | 24 | import sh 25 | from werkzeug.wrappers import Request, Response 26 | from werkzeug.serving import run_simple 27 | from werkzeug.routing import Map, Rule 28 | from werkzeug.exceptions import HTTPException, NotFound, ImATeapot 29 | from werkzeug.utils import redirect 30 | 31 | from ..utils import log 32 | from . import rpc 33 | 34 | STACKHUT_DIR = os.path.abspath('.stackhut') 35 | 36 | def get_req_dir(req_id): 37 | return os.path.join(STACKHUT_DIR, req_id) 38 | 39 | def get_req_file(req_id, fname): 40 | return os.path.join(STACKHUT_DIR, req_id, fname) 41 | 42 | def http_status_code(data): 43 | if type(data) == list: 44 | log.debug("Shit, HTTP status code incorrect") 45 | 46 | if 'error' not in data.get('response', {}): 47 | return 200 48 | 49 | code = data['response']['error']['code'] 50 | 51 | if code == -32600: 52 | return 400 53 | elif code == -32601: 54 | return 404 55 | else: 56 | return 500 57 | 58 | class AbstractBackend: 59 | """A base wrapper wrapper around common IO task state""" 60 | def __init__(self, hutcfg, author): 61 | self.author = author 62 | self.service_short_name = hutcfg.service_short_name(self.author) 63 | os.mkdir(STACKHUT_DIR) if not os.path.exists(STACKHUT_DIR) else None 64 | self.request = {} 65 | log.debug("Starting service {}".format(self.service_short_name)) 66 | 67 | def __enter__(self): 68 | return self 69 | 70 | def __exit__(self, exc_type, exc_val, exc_tb): 71 | pass 72 | 73 | # Interace between backend and runner 74 | @abc.abstractmethod 75 | def get_request(self): 76 | pass 77 | 78 | @abc.abstractmethod 79 | def put_response(self, s): 80 | pass 81 | 82 | # First-stage processing of request/response 83 | def _process_request(self, data): 84 | try: 85 | self.request = json.loads(data.decode('utf-8')) 86 | rpc.add_get_id(self.request) 87 | log.info("Request - {}".format(self.request)) 88 | if ((self.request['service'] != self.service_short_name) and ((self.request['service']+':latest') != self.service_short_name)): 89 | log.warn("Service request ({}) sent to wrong service ({})".format(self.request['service'], self.service_short_name)) 90 | except Exception as e: 91 | _e = rpc.exc_to_json_error(rpc.ParseError(dict(exception=repr(e)))) 92 | return True, _e 93 | else: 94 | return False, self.request 95 | 96 | def _process_response(self, data): 97 | log.info("Response - {}".format(data)) 98 | self.request = {} 99 | return json.dumps(data).encode('utf-8') 100 | 101 | def get_file(self, key): 102 | raise NotImplementedError("IOStore.get_file called") 103 | 104 | @abc.abstractmethod 105 | def put_file(self, fname, req_id='', make_public=False): 106 | pass 107 | 108 | @property 109 | def task_id(self): 110 | return self.request.get('id', None) 111 | 112 | def create_request_dir(self, req_id): 113 | # create a private working dir 114 | req_path = os.path.join(STACKHUT_DIR, req_id) 115 | os.mkdir(req_path) if not os.path.exists(req_path) else None 116 | return req_path 117 | 118 | def del_request_dir(self, req_id): 119 | req_path = os.path.join(STACKHUT_DIR, req_id) 120 | shutil.rmtree(req_path, ignore_errors=True) 121 | 122 | 123 | class LocalRequestServer(threading.Thread): 124 | def __init__(self, port, backend, req_q, resp_q): 125 | super().__init__(daemon=True) 126 | # configure the local server thread 127 | self.port = port 128 | self.req_q = req_q 129 | self.resp_q = resp_q 130 | self.backend = backend 131 | # self.got_req = threading.Event() 132 | self.start() 133 | 134 | # routing 135 | self.url_map = Map([ 136 | Rule('/run', endpoint='run_request'), 137 | Rule('/files', endpoint='run_files'), 138 | ]) 139 | 140 | def run(self): 141 | # start in a new thread 142 | log.info("Started StackHut Request Server - press Ctrl-C to quit") 143 | run_simple('0.0.0.0', self.port, self.local_server) 144 | 145 | @Request.application 146 | def local_server(self, request): 147 | """ 148 | Local webserver running on separate thread for dev usage 149 | Sends msgs to LocalBackend over a pair of shared queues 150 | """ 151 | adapter = self.url_map.bind_to_environ(request.environ) 152 | try: 153 | endpoint, values = adapter.match() 154 | return getattr(self, 'on_' + endpoint)(request, **values) 155 | except HTTPException as e: 156 | return e 157 | 158 | def on_run_request(self, request): 159 | """ 160 | Sends run requests to LocalBackend over a pair of shared queues 161 | """ 162 | (rpc_error, data) = self.backend._process_request(request.data) 163 | if rpc_error: 164 | return self.return_reponse(data) 165 | 166 | task_req = data 167 | self.req_q.put(task_req) 168 | response = self.resp_q.get() 169 | 170 | self.req_q.task_done() 171 | self.resp_q.task_done() 172 | 173 | return self.return_reponse(response) 174 | 175 | def return_reponse(self, data): 176 | return Response(self.backend._process_response(data), 177 | status=http_status_code(data), mimetype='application/json') 178 | 179 | def on_run_files(self, request): 180 | log.debug("In run_files endpoint") 181 | raise ImATeapot() 182 | 183 | 184 | 185 | class LocalBackend(AbstractBackend): 186 | """Mock storage and server system for local testing""" 187 | local_store = "run_result" 188 | 189 | def _get_path(self, name): 190 | return "{}/{}".format(self.local_store, name) 191 | 192 | def __init__(self, hutcfg, author, port, uid_gid=None): 193 | super().__init__(hutcfg, author) 194 | self.uid_gid = uid_gid 195 | 196 | # delete and recreate local_store 197 | shutil.rmtree(self.local_store, ignore_errors=True) 198 | if not os.path.exists(self.local_store): 199 | os.mkdir(self.local_store) 200 | 201 | # configure the local server thread 202 | self.req_q = Queue(1) 203 | self.resp_q = Queue(1) 204 | self.server = LocalRequestServer(port, self, self.req_q, self.resp_q) 205 | 206 | def __exit__(self, exc_type, exc_val, exc_tb): 207 | log.debug("Shutting down Local backend") 208 | # wait for queues to empty 209 | self.req_q.join() 210 | self.resp_q.join() 211 | 212 | # change the results owner 213 | if self.uid_gid is not None: 214 | sh.chown('-R', self.uid_gid, self.local_store) 215 | 216 | def get_request(self): 217 | return self.req_q.get() 218 | 219 | def put_response(self, data): 220 | self.resp_q.put(data) 221 | 222 | def _process_response(self, _data): 223 | """For local wrap up in a response dict""" 224 | data = dict(response=_data) 225 | return super()._process_response(data) 226 | 227 | def put_file(self, fname, req_id='', make_public=True): 228 | """Put file into a subdir keyed by req_id in local store""" 229 | if req_id == '': 230 | req_fname = fname 231 | else: 232 | req_fname = get_req_file(req_id, fname) 233 | 234 | local_store_dir = self._get_path(req_id) 235 | 236 | os.mkdir(local_store_dir) if not os.path.exists(local_store_dir) else None 237 | shutil.copy(req_fname, local_store_dir) 238 | return os.path.join(local_store_dir, fname) 239 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/runtime/runner.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import signal 15 | import threading 16 | 17 | from . import rpc 18 | from .runtime_server import RuntimeServer 19 | from ..utils import log 20 | 21 | shim_cmds = { 22 | 'python': ['/usr/bin/env', 'python3', 'runner.py'], 23 | 'python2': ['/usr/bin/env', 'python2', 'runner.py'], 24 | 'nodejs': ['/usr/bin/env', 'node', '--es_staging', 'runner.js'] 25 | } 26 | 27 | def sigterm_handler(signo, frame): 28 | log.debug("Got shutdown signal".format(signo)) 29 | raise KeyboardInterrupt 30 | 31 | class ServiceRunner: 32 | """Runs a service""" 33 | def __init__(self, backend, hutcfg): 34 | log.debug('Starting Service Runner') 35 | self.backend = backend 36 | self.hutcfg = hutcfg 37 | # select the stack 38 | self.shim_cmd = shim_cmds.get(self.hutcfg.stack) 39 | if self.shim_cmd is None: 40 | raise RuntimeError("Unknown stack - {}".format(self.hutcfg.stack)) 41 | 42 | # init the local runtime service 43 | self.runtime_server = RuntimeServer(backend) 44 | # init the rpc server 45 | self.rpc = rpc.StackHutRPC(self.backend, self.shim_cmd) 46 | 47 | assert threading.current_thread() == threading.main_thread() 48 | signal.signal(signal.SIGTERM, sigterm_handler) 49 | signal.signal(signal.SIGINT, sigterm_handler) 50 | 51 | def __enter__(self): 52 | return self 53 | 54 | def __exit__(self, exc_type, exc_val, exc_tb): 55 | """clean the system, write all output data and exit""" 56 | log.debug('Shutting down service runner') 57 | 58 | def run(self): 59 | # error_count = 0 60 | 61 | # setup the run contexts 62 | with self.backend, self.runtime_server, self.rpc: 63 | while True: 64 | try: 65 | # get the request 66 | task_req = self.backend.get_request() 67 | # make the internal rpc call 68 | task_resp = self.rpc.call(task_req) 69 | # send the response out to the storage backend 70 | self.backend.put_response(task_resp) 71 | except KeyboardInterrupt: 72 | break 73 | 74 | if not self.hutcfg.persistent: 75 | break 76 | 77 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/runtime/runtime_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | StackHut Runtime library, accessible over JSON-RPC 16 | """ 17 | import threading 18 | 19 | import requests 20 | import sh 21 | from werkzeug.wrappers import Request, Response 22 | from werkzeug.serving import run_simple 23 | from jsonrpc import JSONRPCResponseManager, dispatcher 24 | 25 | from ..utils import log 26 | from . import rpc, backends 27 | 28 | backend = None 29 | 30 | 31 | class RuntimeServer(threading.Thread): 32 | def __init__(self, _backend): 33 | super().__init__(daemon=True) 34 | global backend 35 | backend = _backend 36 | 37 | @Request.application 38 | def application(self, request): 39 | log.debug("Got helper request - {}".format(request.data)) 40 | response = JSONRPCResponseManager.handle(request.data, dispatcher) 41 | return Response(response.json, mimetype='application/json') 42 | 43 | def run(self): 44 | # start in a new thread 45 | log.debug("Starting StackHut helper-server") 46 | run_simple('localhost', 4000, self.application, threaded=True) 47 | 48 | def __enter__(self): 49 | self.start() 50 | return self 51 | 52 | def __exit__(self, exc_type, exc_val, exc_tb): 53 | pass 54 | 55 | ############################################################################### 56 | # Runtime Functions 57 | 58 | 59 | @dispatcher.add_method 60 | def get_stackhut_user(req_id): 61 | auth = backend.request.get('auth', None) 62 | return auth['username'] if auth else '' 63 | 64 | 65 | @dispatcher.add_method 66 | def get_service_author(req_id): 67 | return backend.author 68 | 69 | 70 | @dispatcher.add_method 71 | def is_author(req_id): 72 | return (get_stackhut_user(req_id) == get_service_author(req_id)) 73 | 74 | 75 | @dispatcher.add_method 76 | def put_file(req_id, fname, make_public=True): 77 | return backend.put_file(fname, req_id, make_public) 78 | 79 | 80 | @dispatcher.add_method 81 | def get_file(req_id, key): 82 | return backend.get_file(key) 83 | 84 | 85 | # File upload / download helpers 86 | @dispatcher.add_method 87 | def download_file(req_id, url, fname=None): 88 | """from http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py""" 89 | fname = url.split('/')[-1] if fname is None else fname 90 | req_fname = backends.get_req_file(req_id, fname) 91 | log.info("Downloading file {} from {}".format(fname, url)) 92 | r = requests.get(url, stream=True) 93 | with open(req_fname, 'wb') as f: 94 | for chunk in r.iter_content(chunk_size=1024): 95 | if chunk: # filter out keep-alive new chunks 96 | f.write(chunk) 97 | return fname 98 | 99 | 100 | @dispatcher.add_method 101 | def run_command(req_id, cmd, stdin=''): 102 | try: 103 | cmd_run = sh.Command(cmd) 104 | output = cmd_run(_in=stdin) 105 | except sh.ErrorReturnCode as e: 106 | raise rpc.NonZeroExitError(e.exit_code, e.stderr) 107 | return output 108 | -------------------------------------------------------------------------------- /stackhut_toolkit/common/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | import sys 16 | import os 17 | from colorlog import ColoredFormatter 18 | 19 | #################################################################################################### 20 | # App Config 21 | # global constants 22 | CONTRACTFILE = '.api.json' 23 | IDLFILE = 'api.idl' 24 | SERVER_URL = "https://api.stackhut.com/" 25 | IN_CONTAINER = os.path.exists('/workdir') 26 | VERBOSE = False 27 | ROOT_DIR = os.getcwd() 28 | 29 | def change_root_dir(dir): 30 | global ROOT_DIR 31 | os.chdir(dir) 32 | ROOT_DIR = os.getcwd() 33 | 34 | # Logging 35 | # LOGFILE = '.stackhut.log' 36 | logging.getLogger().disabled = True 37 | logging.getLogger('werkzeug').disabled = True 38 | 39 | log = logging.getLogger('stackhut') 40 | def setup_logging(verbose_mode): 41 | global VERBOSE 42 | global log 43 | VERBOSE = verbose_mode 44 | log.propagate = False 45 | log.setLevel(logging.DEBUG if verbose_mode else logging.INFO) 46 | 47 | logFormatter = ColoredFormatter( 48 | '%(blue)s%(asctime)s%(reset)s [%(log_color)s%(levelname)-5s%(reset)s] %(message)s', 49 | datefmt='%H:%M:%S', 50 | reset=True, 51 | log_colors={ 52 | 'DEBUG': 'cyan', 53 | 'INFO': 'green', 54 | 'WARNING': 'yellow', 55 | 'ERROR': 'red', 56 | 'CRITICAL': 'red,bg_white', 57 | }, 58 | secondary_log_colors={}, 59 | style='%' 60 | ) 61 | 62 | # file output 63 | # fileHandler = logging.FileHandler(LOGFILE, mode='w') 64 | # fileHandler.setFormatter(logFormatter) 65 | # log.addHandler(fileHandler) 66 | 67 | # console 68 | consoleHandler = logging.StreamHandler(stream=sys.stdout) 69 | consoleHandler.setFormatter(logFormatter) 70 | log.addHandler(consoleHandler) 71 | -------------------------------------------------------------------------------- /stackhut_toolkit/manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Minimal StackHut service orchestration tool 16 | """ 17 | import json 18 | import os 19 | import time 20 | from threading import Thread 21 | 22 | from multiprocessing import Process, Manager 23 | 24 | 25 | 26 | 27 | 28 | if __name__ == '__main__': 29 | with Manager() as manager: 30 | d = manager.dict() 31 | l = manager.list(range(10)) 32 | 33 | p = Process(target=f, args=(d, l)) 34 | p.start() 35 | p.join() 36 | 37 | print(d) 38 | print(l) 39 | 40 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/common/.gitignore: -------------------------------------------------------------------------------- 1 | .stackhut.log 2 | *.json 3 | !test_request.json 4 | run_result 5 | .stackhut 6 | .Dockerfile 7 | 8 | # ide 9 | *.iml 10 | .idea 11 | 12 | # Byte-compiled / optimized / DLL files 13 | __pycache__/ 14 | *.py[cod] 15 | 16 | # C extensions 17 | *.so 18 | 19 | # Distribution / packaging 20 | .Python 21 | env/ 22 | build/ 23 | develop-eggs/ 24 | dist/ 25 | downloads/ 26 | eggs/ 27 | .eggs/ 28 | lib/ 29 | lib64/ 30 | parts/ 31 | sdist/ 32 | var/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *,cover 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/common/Hutfile.yaml: -------------------------------------------------------------------------------- 1 | # Service name (a combination of lower case letters, numbers, and dashes) 2 | name: {{ scaffold.name }} 3 | 4 | # Service description 5 | description: Awesome service in {{ scaffold.stack.name }} 6 | 7 | # GitHub source repo link (optional) 8 | # github_url: www.github.com/... 9 | 10 | # The service dependencies, in terms of the base OS and language runtime 11 | baseos: {{ scaffold.baseos.name }} 12 | stack: {{ scaffold.stack.name }} 13 | 14 | # Persist the service between requests 15 | persistent: True 16 | 17 | # Restrict service access to authenticated users 18 | private: False 19 | 20 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/common/README.md: -------------------------------------------------------------------------------- 1 | # {{ scaffold.name }} Service 2 | 3 | _TODO_ - describe service 4 | 5 | ## Usage 6 | * Look at the API (defined in `api.idl`) to see what functions are available, friend. 7 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/common/api.idl: -------------------------------------------------------------------------------- 1 | interface Default { 2 | // add 2 integers and return the result 3 | add(x int, y int) int 4 | } 5 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/common/test_request.json: -------------------------------------------------------------------------------- 1 | { 2 | "service": "{{ scaffold.author}}/{{ scaffold.name }}", 3 | "request": { 4 | "method": "add", 5 | "params": [1, 2] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/nodejs/app.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | // {{ scaffold.name }} service 3 | let stackhut = require('./stackhut'); 4 | 5 | // create each service as either an ES6 class or an object of functions 6 | class Default extends stackhut.Service { 7 | constructor() { 8 | super(); 9 | // empty 10 | } 11 | 12 | add(x, y){ 13 | let res = x + y; 14 | return Promise.resolve(res); 15 | } 16 | } 17 | 18 | // export the services here 19 | module.exports = { 20 | Default : new Default() 21 | }; 22 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/python/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | {{ scaffold.name }} Service 5 | """ 6 | import stackhut 7 | 8 | class Default(stackhut.Service): 9 | def add(self, x, y): 10 | return x + y 11 | 12 | # export the services 13 | SERVICES = {"Default": Default()} 14 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/scaffold/python/requirements.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/shims/nodejs/runner.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | "use strict"; 3 | // Copyright 2015 StackHut Ltd. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | // any 1st & 3rd-party modules here 18 | let fs = require('fs'); 19 | let path = require('path'); 20 | let process = require('process'); 21 | // load the app to call into 22 | let app = require('./app'); 23 | let stackhut = require('./stackhut'); 24 | // let util = require('util'); 25 | // console.log('app - \n\t', util.inspect(app, false, null)); 26 | // console.log('stackhut - \n\t', util.inspect(stackhut, false, null)); 27 | 28 | /////////////////////////////////////////////////////////////////////////////// 29 | // Utils 30 | const REQ_JSON = '.req.json'; 31 | const RESP_JSON = '.resp.json'; 32 | 33 | // simple error handling 34 | function gen_error(code, msg, _data) { 35 | let data = typeof _data !== 'undefined' ? _data : {}; 36 | // b = typeof b !== 'undefined' ? b : 1; 37 | return { error: code, msg: msg, data: data }; 38 | } 39 | 40 | // custom write func as bloody Node can't write to a named pipe otherwise! 41 | function sync_write_resp(resp) { 42 | let buf = new Buffer(JSON.stringify(resp)); 43 | let fd = fs.openSync(RESP_JSON, 'w'); 44 | fs.writeSync(fd, buf, 0, buf.length, -1); 45 | fs.closeSync(fd) 46 | } 47 | 48 | /////////////////////////////////////////////////////////////////////////////// 49 | // Main Run function 50 | function run(req) { 51 | // tell the client helper the current taskid 52 | stackhut.req_id = req['req_id']; 53 | 54 | let ms = req['method'].split('.'); 55 | let iface_name = ms[0]; 56 | let func_name = ms[1]; 57 | let params = req['params']; 58 | 59 | // get the iface, then the func, and call it dyn 60 | if (iface_name in app) { 61 | let iface_impl = app[iface_name]; 62 | 63 | if (func_name in iface_impl) { 64 | let func_impl = iface_impl[func_name]; 65 | 66 | // TODO - how to use spread with dyn call into object? 67 | // return func_impl(...params) 68 | return func_impl.apply(iface_impl, params) 69 | .catch(function(result) { 70 | if (typeof result == 'string') { 71 | return Promise.reject(gen_error(-32602, result)) 72 | } else { 73 | return Promise.reject(gen_error(-32602, result[0], result[1])) 74 | } 75 | }) 76 | } 77 | else { return Promise.reject(gen_error(-32601, 'Method not found')); } 78 | } 79 | else { return Promise.reject(gen_error(-32601, 'Service not found')); } 80 | } 81 | 82 | 83 | // top-level error handling 84 | process.on('uncaughtException', function(err) { 85 | console.log('Uncaught Exception - %s', err); 86 | let resp = gen_error(-32000, err.toString()); 87 | sync_write_resp(resp); 88 | process.exit(0); 89 | }); 90 | 91 | //process.on('SIGTERM', function() { 92 | // //console.log('Received shutdown signal'); 93 | // process.exit(0); 94 | //}); 95 | 96 | // mutual recursion to handle processing req->resp 97 | // Thank god for TCO in ES6 98 | function process_resp(resp) { 99 | process.chdir(stackhut.root_dir); 100 | // save the json resp 101 | sync_write_resp(resp); 102 | process_req(); 103 | } 104 | 105 | function process_req() { 106 | // open the json req 107 | let req = JSON.parse(fs.readFileSync(REQ_JSON, 'utf8')); 108 | 109 | process.chdir(path.join('.stackhut', req['req_id'])); 110 | 111 | // run the command sync/async and then return the result or error 112 | run(req) 113 | .then(function(resp) { 114 | process_resp({ result: resp }); 115 | }) 116 | .catch(function(err) { 117 | process_resp(err); 118 | }); 119 | } 120 | 121 | // start the loop 122 | process_req(); 123 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/shims/nodejs/stackhut.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | // Copyright 2015 StackHut Ltd. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | let request = require('request'); 17 | let url = "http://localhost:4000/jsonrpc"; 18 | 19 | let id_val = 0; 20 | module.exports.req_id = null; 21 | module.exports.root_dir = __dirname; 22 | 23 | module.exports.Service = class { 24 | constructor() { 25 | // empty 26 | }; 27 | 28 | shutdown() { 29 | return Promise.resolve(null); 30 | }; 31 | 32 | preBatch() { 33 | return Promise.resolve(null); 34 | }; 35 | 36 | postBatch() { 37 | return Promise.resolve(null); 38 | }; 39 | 40 | preRequest() { 41 | return Promise.resolve(null); 42 | }; 43 | 44 | postRequest() { 45 | return Promise.resolve(null); 46 | }; 47 | }; 48 | 49 | /////////////////////////////////////////////////////////////////////////////// 50 | // Runtime Lib 51 | function make_call(method, ...params) { 52 | 53 | //let params = [].slice.call(arguments, 1); 54 | params.unshift(module.exports.req_id); 55 | 56 | let payload = { 57 | method: method, 58 | params: params, 59 | jsonrpc: '2.0', 60 | id: id_val 61 | }; 62 | 63 | return new Promise(function(resolve, reject) { 64 | request({ 65 | url: url, 66 | method: 'POST', 67 | body: payload, 68 | json: true 69 | }, 70 | function(error, response, body) { 71 | if(!error && response.statusCode >= 200 && response.statusCode < 300) { 72 | id_val += 1; 73 | if ('result' in body) { 74 | resolve(body['result']); 75 | } else { reject(body['error']) } 76 | } else { 77 | reject('error: '+ response.statusCode + error) 78 | } 79 | } 80 | ) 81 | }) 82 | } 83 | 84 | // stackhut library functions 85 | module.exports.get_stackhut_user = function() { 86 | return make_call('get_stackhut_user') 87 | }; 88 | 89 | module.exports.get_service_author = function() { 90 | return make_call('get_service_author') 91 | }; 92 | 93 | module.exports.is_author = function() { 94 | return make_call('is_author') 95 | }; 96 | 97 | module.exports.put_file = function(fname, make_public) { 98 | let _make_public = typeof make_public !== 'undefined' ? make_public : true; 99 | return make_call('put_file', fname, _make_public) 100 | }; 101 | 102 | module.exports.get_file = function(key) { 103 | return make_call('get_file', key) 104 | }; 105 | 106 | module.exports.download_file = function(url, fname) { 107 | let _fname = typeof fname !== 'undefined' ? fname : null; 108 | return make_call('download_file', url, _fname) 109 | }; 110 | 111 | module.exports.run_command = function(cmd, stdin) { 112 | let _stdin = typeof stdin !== 'undefined' ? stdin : ''; 113 | return make_call('run_command', _cmd) 114 | }; 115 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/shims/python/runner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2015 StackHut Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """ 17 | Demo StackHut service 18 | """ 19 | import json 20 | import os 21 | import signal 22 | import sys 23 | import stackhut 24 | from app import SERVICES 25 | 26 | def gen_error(code, msg='', data=None): 27 | return dict(error=code, msg=msg, data=data) 28 | 29 | def run(req): 30 | # tell the client helper the current taskid 31 | stackhut.req_id = req['req_id'] 32 | 33 | iface_name, func_name = req['method'].split('.') 34 | params = req['params'] 35 | 36 | if iface_name in SERVICES: 37 | iface_impl = SERVICES[iface_name] 38 | 39 | try: 40 | func = getattr(iface_impl, func_name) 41 | except AttributeError: 42 | return gen_error(-32601) 43 | 44 | # iface_impl.preRequest() 45 | # result = func(*params) if params else func() 46 | # iface_impl.postRequest() 47 | 48 | try: 49 | result = func(*params) if params else func() 50 | except stackhut.ServiceError as e: 51 | return gen_error(-32002, e.msg, e.data) 52 | 53 | return dict(result=result) 54 | else: 55 | return gen_error(-32601) 56 | 57 | def sigterm_handler(signo, frame): 58 | print("Received shutdown signal".format(signo)) 59 | sys.exit(0) 60 | 61 | if __name__ == "__main__": 62 | signal.signal(signal.SIGTERM, sigterm_handler) 63 | 64 | try: 65 | while True: 66 | # open the input 67 | with open(".req.json", "r") as f: 68 | req = json.loads(f.read()) 69 | 70 | os.chdir(os.path.join('.stackhut', req['req_id'])) 71 | 72 | # run the command 73 | try: 74 | resp = run(req) 75 | except Exception as e: 76 | resp = gen_error(-32603, repr(e)) 77 | 78 | os.chdir(stackhut.root_dir) 79 | 80 | # save the output 81 | with open(".resp.json", "w") as f: 82 | f.write(json.dumps(resp)) 83 | 84 | except Exception as e: 85 | print(repr(e)) 86 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/shims/python/stackhut.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | import requests 16 | import json 17 | import os 18 | 19 | url = "http://localhost:4000/jsonrpc" 20 | headers = {'content-type': 'application/json'} 21 | 22 | id_val = 0 23 | req_id = None 24 | 25 | class Service: 26 | def __init__(self): 27 | pass 28 | 29 | def shutdown(self): 30 | pass 31 | 32 | def preBatch(self): 33 | pass 34 | 35 | def postBatch(self): 36 | pass 37 | 38 | def preRequest(self): 39 | pass 40 | 41 | def postRequest(self): 42 | pass 43 | 44 | class ServiceError(Exception): 45 | def __init__(self, msg, data=None): 46 | self.msg = msg 47 | self.data = data 48 | 49 | 50 | ############################################################################### 51 | # Runtime Lib 52 | def make_call(method, *_params): 53 | global id_val 54 | global req_id 55 | 56 | params = list(_params) 57 | params.insert(0, req_id) 58 | 59 | payload = { 60 | 'method': method, 61 | 'params': params, 62 | 'jsonrpc': '2.0', 63 | 'id': id_val, 64 | } 65 | 66 | response = requests.post(url, data=json.dumps(payload), headers=headers).json() 67 | 68 | id_val += 1 69 | 70 | if 'result' in response: 71 | return response['result'] 72 | else: 73 | raise RuntimeError(response['error']) 74 | 75 | # stackhut fields 76 | root_dir = os.getcwd() 77 | in_container = True if os.path.exists('/workdir') else False 78 | 79 | # stackhut library functions 80 | def get_stackhut_user(): 81 | return make_call('get_stackhut_user') 82 | 83 | def get_service_author(): 84 | return make_call('get_service_author') 85 | 86 | def is_author(): 87 | return make_call('is_author') 88 | 89 | def put_file(fname, make_public=True): 90 | return make_call('put_file', fname, make_public) 91 | 92 | def get_file(key): 93 | return make_call('get_file', key) 94 | 95 | def download_file(url, fname=None): 96 | return make_call('download_file', url, fname) 97 | 98 | def run_command(cmd, stdin=''): 99 | return make_call('run_command', cmd) 100 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/templates/Docker-runtime.txt: -------------------------------------------------------------------------------- 1 | FROM {{ service.usercfg.username }}/{{ service.hutcfg.from_image }}:latest 2 | {# MAINTAINER "{{ service.usercfg.email }}" <{{ service.usercfg.username }}> #} 3 | {# LABEL description="{{ service.hutcfg.description }}" #} 4 | 5 | # OK - now setup stackhut runner 6 | # following toolkit.commands never cached 7 | 8 | # setup the entrypoint (using -vv for now) 9 | ENTRYPOINT ["/usr/bin/stackhut", "-vv", "run"] 10 | 11 | # install stackhut app 12 | RUN pip3 install --no-cache-dir --compile stackhut # cache bust - {{ service.build_date }} 13 | 14 | RUN echo "Starting..." && \ # cache bust - {{ service.build_date }} 15 | wget http://s3-eu-west-1.amazonaws.com/stackhut-releases/stackhut-latest.linux-x86_64.tar.gz && \ 16 | gunzip ./stackhut-latest.linux-x86_64.tar.gz && \ 17 | tar -xvf ./stackhut-latest.linux-x86_64.tar -C / && \ 18 | echo "...done" && exit 19 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/templates/Dockerfile-baseos.txt: -------------------------------------------------------------------------------- 1 | FROM {{ baseos.name|lower }}:{{ baseos.tag }} 2 | MAINTAINER "StackHut" 3 | LABEL description="{{ baseos.description }}" 4 | 5 | ADD https://bootstrap.pypa.io/get-pip.py / 6 | 7 | # update the distro and commit 8 | RUN echo "Starting..." && \ 9 | {% for cmd in baseos.setup_cmds() -%} 10 | {{ cmd }} && \ 11 | {% endfor -%} 12 | echo "...done" && exit 0 13 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/templates/Dockerfile-service.txt: -------------------------------------------------------------------------------- 1 | FROM stackhut/{{ service.hutcfg.from_image }}:latest 2 | {# MAINTAINER "{{ service.usercfg.email }}" <{{ service.usercfg.username }}> -#} 3 | {# LABEL description="{{ service.hutcfg.description }}" -#} 4 | 5 | {% if service.hutcfg.os_deps %} 6 | # install OS packages needed for service 7 | RUN echo "Starting..." && \ 8 | {% for cmd in service.baseos.install_os_pkg(service.hutcfg.os_deps) -%} 9 | {{ cmd }} && \ 10 | {% endfor -%} 11 | echo "...done" && exit 12 | {% endif %} 13 | 14 | WORKDIR /workdir 15 | 16 | {% if service.dev %} 17 | # install stackhut-runner from dev branch 18 | RUN echo "Starting..." && \ 19 | pip3 install --no-cache-dir --compile https://github.com/StackHut/stackhut-common/archive/master.zip && \ 20 | pip3 install --no-cache-dir --compile https://s3-eu-west-1.amazonaws.com/stackhut-releases/dev/stackhut_runner-1.0.dev0-py3-none-any.whl && \ 21 | echo "...done" && exit 22 | {% else %} 23 | # install stackhut-runner from release branch 24 | RUN pip3 install --no-cache-dir --compile https://s3-eu-west-1.amazonaws.com/stackhut-releases/stackhut_runner-1.0.dev0-py3-none-any.whl 25 | {% endif %} 26 | 27 | {% if service.stack.service_package_files %} 28 | # install lang packages needed for stack 29 | COPY {{ service.stack.service_package_files }} ./ 30 | RUN {{ service.stack.install_service_packages() }} 31 | {% endif %} 32 | 33 | # copy all source files and dirs across 34 | COPY Hutfile.yaml .api.json {{ service.stack.entrypoint }} {{ service.stack.shim_files|join(' ') }} {{ service.hutcfg.files|join(' ') }} ./ 35 | {% for d in service.hutcfg.dirs -%} 36 | COPY {{ d }} ./{{ d }} 37 | {% endfor -%} 38 | 39 | # any other Docker toolkit.commands 40 | {% for cmd in service.hutcfg.docker_cmds -%} 41 | {{ cmd }} 42 | {% endfor -%} 43 | 44 | {## cache bust - {{ service.build_date }}#} 45 | 46 | {# 47 | # setup the entrypoint (using -v for now) 48 | ENTRYPOINT ["/usr/bin/python3", "/usr/bin/stackhut", "-v", "run"] 49 | #} 50 | -------------------------------------------------------------------------------- /stackhut_toolkit/res/templates/Dockerfile-stack.txt: -------------------------------------------------------------------------------- 1 | FROM stackhut/{{ baseos.name|lower }}:latest 2 | MAINTAINER "StackHut" 3 | LABEL description="{{ stack.description }}" 4 | 5 | {% if stack_cmds %} 6 | # install OS packages needed for stack 7 | RUN echo "Starting..." && \ 8 | {% for cmd in stack_cmds -%} 9 | {{ cmd }} && \ 10 | {% endfor -%} 11 | echo "...done" && exit 12 | {% endif %} 13 | 14 | {% if stack.stack_packages %} 15 | # install lang packages needed for stack 16 | RUN {{ stack.install_stack_packages() }} 17 | {% endif %} 18 | -------------------------------------------------------------------------------- /stackhut_toolkit/toolkit_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 StackHut Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import threading 15 | import sys 16 | import os 17 | import json 18 | import itertools 19 | import time 20 | from queue import Queue 21 | import urllib.parse 22 | import requests 23 | from .common.utils import log 24 | from .common import utils 25 | 26 | 27 | # names to export 28 | __all__ = ['stackhut_api_call', 'stackhut_api_user_call', 'keen_client', 'get_res_path', 'Spinner'] 29 | 30 | # Setup app paths - this is unique for each stackhut package 31 | sys_dir = os.path.dirname(sys.executable) if getattr(sys, 'frozen', False) else os.path.dirname(__file__) 32 | res_dir = os.path.normpath(os.path.join(sys_dir, './res')) 33 | 34 | def get_res_path(res_name): 35 | return os.path.join(res_dir, res_name) 36 | 37 | ################################################################################################### 38 | # StackHut server comms 39 | json_header = {'content-type': 'application/json'} 40 | 41 | 42 | def stackhut_api_call(endpoint, msg, secure=True, return_json=True): 43 | url = urllib.parse.urljoin(utils.SERVER_URL, endpoint) 44 | log.debug("Calling Stackhut Server at {} with \n\t{}".format(url, json.dumps(msg))) 45 | r = requests.post(url, data=json.dumps(msg), headers=json_header) 46 | 47 | if r.status_code == requests.codes.ok: 48 | return r.json() if return_json else r.text 49 | else: 50 | log.error("Error {} talking to Stackhut Server".format(r.status_code)) 51 | log.error(r.text) 52 | r.raise_for_status() 53 | 54 | def stackhut_api_user_call(endpoint, _msg, usercfg): 55 | msg = _msg.copy() 56 | msg['auth'] = dict(username=usercfg.username, hash=usercfg['hash']) 57 | return stackhut_api_call(endpoint, msg) 58 | 59 | 60 | 61 | ################################################################################################### 62 | # Keen analytlics 63 | class KeenClient(threading.Thread): 64 | project_id = '559f866f96773d25d47419f6' 65 | write_key = 'abd65ad8684753678eabab1f1c536b36a70704e6c4f10bcfe928c10ec859edb1d0366f3fad9b7794b0' \ 66 | 'eeab9825a27346e0186e2e062f76079708b66ddfca7ecc82b8db23062f8cd2e4f6a961d8d2ea23b22f' \ 67 | 'c9aae1387514da6d46cdbebec2d15c9167d401963ee8f96b00e06acf4e48' 68 | keen_url = "https://api.keen.io/3.0/projects/{project_id}/events/{{event_collection}}?" \ 69 | "api_key={write_key}".format(project_id=project_id, write_key=write_key) 70 | 71 | def __init__(self, *args, **kwargs): 72 | super().__init__(*args, **kwargs) 73 | self.send_analytics = False 74 | self.analytics_ids = None 75 | self.queue = Queue() 76 | 77 | def start(self, usercfg): 78 | self.send_analytics = usercfg.send_analytics 79 | if self.send_analytics: 80 | log.debug("User analytics enabled") 81 | self.analytics_ids = usercfg.analytics_ids 82 | super().start() 83 | else: 84 | log.debug("User analytics disabled") 85 | 86 | def run(self): 87 | while True: 88 | (endpoint, msg) = self.queue.get() 89 | msg.update(self.analytics_ids) 90 | try: 91 | log.debug("Sending analytics msg to {}".format(endpoint)) 92 | # log.debug("Analytics msg - {}".format(msg)) 93 | url = self.keen_url.format(event_collection=endpoint) 94 | r = requests.post(url, data=json.dumps(msg), headers=json_header, timeout=2) 95 | if not (r.status_code == requests.codes.created and r.json().get('created')): 96 | log.debug("{} - {}".format(r.status_code, r.text())) 97 | raise IOError() 98 | except: 99 | log.debug("Failed sending analytics msg to '{}'".format(endpoint)) 100 | finally: 101 | self.queue.task_done() 102 | 103 | def send(self, endpoint, msg): 104 | if self.send_analytics: 105 | self.queue.put((endpoint, msg)) 106 | 107 | def shutdown(self): 108 | if self.send_analytics: 109 | self.queue.join() 110 | 111 | keen_client = KeenClient(daemon=True) 112 | 113 | 114 | class Spinner(threading.Thread): 115 | """A simple console spinner to use with long-running tasks""" 116 | 117 | spin_interval = 0.5 118 | dot_interval = 10 119 | dot_max = int(dot_interval / spin_interval) 120 | 121 | def __init__(self): 122 | super().__init__(daemon=True) 123 | self.spinning = threading.Event() 124 | self.spinner = itertools.cycle(['-', '\\', '|', '/']) 125 | 126 | def __enter__(self): 127 | self.spinning.set() 128 | self.start() 129 | return self 130 | 131 | def __exit__(self, exc_type, exc_val, exc_tb): 132 | self.spinning.clear() 133 | 134 | def run(self): 135 | dot_count = 0 136 | 137 | while self.spinning.is_set(): 138 | sys.stdout.write(next(self.spinner)) # write the next character 139 | sys.stdout.flush() # flush stdout buffer (actual character display) 140 | sys.stdout.write('\b') # erase the last written char 141 | time.sleep(self.spin_interval) 142 | dot_count += 1 143 | if dot_count >= self.dot_max: 144 | sys.stdout.write('.') # write the next character 145 | dot_count = 0 146 | 147 | sys.stdout.write('\n') 148 | 149 | def stop(self): 150 | self.spinning.clear() 151 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .test_toolkit import * 4 | 5 | -------------------------------------------------------------------------------- /tests/test_toolkit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_stackhut 6 | ---------------------------------- 7 | 8 | Tests for `stackhut` module. 9 | """ 10 | 11 | import unittest 12 | import os 13 | import time 14 | import shutil 15 | import sh 16 | 17 | from stackhut_toolkit.common import config 18 | from stackhut_toolkit.common.runtime import rpc 19 | from stackhut_toolkit.builder import get_docker, bases, stacks 20 | from stackhut_client import client 21 | 22 | 23 | def copy_config(suffix): 24 | src = os.path.expanduser(os.path.join('~', '.stackhut.cfg.{}'.format(suffix))) 25 | dest = os.path.expanduser(os.path.join('~', '.stackhut.cfg')) 26 | shutil.copy(src, dest) 27 | 28 | 29 | class SHToolkitTest(unittest.TestCase): 30 | # def __init__(self): 31 | # super().__init__() 32 | # self.out = None 33 | 34 | def run_toolkit(self, subcmd, _args=None, server=None, verbose=False, **kwargs): 35 | def do_args(args): 36 | a = (x[1] if (type(x) is tuple) and x[0] else x for x in args) 37 | b = (x for x in a if x and type(x) is not tuple) 38 | return list(b) 39 | 40 | _args = _args if _args else [] 41 | args = do_args(['-v', (server, '-s'), (server, server), subcmd] + _args) 42 | print(kwargs) 43 | if verbose: 44 | self.out = sh.stackhut(args, _out=lambda x: print(x.strip()), **kwargs) 45 | else: 46 | self.out = sh.stackhut(args, **kwargs) 47 | 48 | 49 | self.assertEqual(0, self.out.exit_code) 50 | 51 | return self.out 52 | 53 | def assertInStdout(self, substring, msg=None): 54 | self.assertIn(substring, self.out.stdout.decode(), msg) 55 | 56 | 57 | # asserts taken from https://github.com/iLoveTux/CLITest 58 | def assertFileExists(self, filename, msg=None): 59 | """Return True if filename exists and is a file, otherwise raise an 60 | AssertionError with msg""" 61 | if msg is None: 62 | msg = "An assertion Failed, file '{}' ".format(filename) 63 | msg += "does not exist or is not a file." 64 | if not (os.path.exists(filename)) or not (os.path.isfile(filename)): 65 | raise AssertionError(msg) 66 | return True 67 | 68 | def assertFileNotExists(self, filename, msg=None): 69 | """Return True if filename does not exist or is not a file, 70 | otherwise raise an AssertionError with msg""" 71 | if msg is None: 72 | msg = "An assertion Failed, file '{}' exists.".format(filename) 73 | if os.path.exists(filename) and os.path.isfile(filename): 74 | raise AssertionError(msg) 75 | return True 76 | 77 | def assertDirectoryExists(self, path, msg=None): 78 | """Return True if path exists and is a directory, otherwise 79 | raise an AssertionError with msg""" 80 | if msg is None: 81 | msg = "An assertion Failed, directory '{}' ".format(path) 82 | msg += "does not exist or is not a directory." 83 | if not (os.path.exists(path)) or not (os.path.isdir(path)): 84 | raise AssertionError(msg) 85 | return True 86 | 87 | def assertDirectoryNotExists(self, path, msg=None): 88 | """Return True if path does not exist or is not a directory, 89 | otherwise raise an AssertionError with msg""" 90 | if msg is None: 91 | msg = "An assertion Failed, directory '{}' exists.".format(path) 92 | if os.path.exists(path) and os.path.isdir(path): 93 | raise AssertionError(msg) 94 | return True 95 | 96 | 97 | class TestToolkit1User(SHToolkitTest): 98 | backup_file = config.UserCfg.config_fpath + '.bak' 99 | 100 | @classmethod 101 | def setUpClass(cls): 102 | copy_config('mands') 103 | 104 | @unittest.skip('disabled for now') 105 | def test_1_login(self): 106 | self.run_toolkit('login', verbose=True) 107 | 108 | usercfg = config.UserCfg() 109 | self.assertIn('username', usercfg) 110 | self.assertIn('hash', usercfg) 111 | 112 | def test_2_info(self): 113 | out = self.run_toolkit('info') 114 | self.assertInStdout('username', 'User logged in') 115 | 116 | @unittest.skip('disabled for now') 117 | def test_3_logout(self): 118 | self.run_toolkit('logout') 119 | usercfg = config.UserCfg() 120 | self.assertNotIn('username', usercfg) 121 | self.assertNotIn('hash', usercfg) 122 | 123 | @classmethod 124 | def tearDownClass(cls): 125 | pass 126 | 127 | 128 | class TestToolkit2StackBuild(SHToolkitTest): 129 | def setUp(self): 130 | self.docker = get_docker() 131 | copy_config('stackhut') 132 | 133 | def check_image(self, image_name, dirs): 134 | """check docker build dir and docker image exists""" 135 | images = self.docker.client.images("{}/{}".format('stackhut', image_name)) 136 | self.assertGreater(len(images), 0) 137 | self.assertIn(image_name, dirs) 138 | 139 | def test_stackbuild(self): 140 | out = self.run_toolkit('stackbuild', ['--outdir', 'test-stackbuild'], verbose=True) 141 | os.chdir('test-stackbuild') 142 | dirs = {d for d in os.listdir('.') if os.path.isdir(d)} 143 | 144 | # check docker build dir and docker image exists 145 | [self.check_image(b.name, dirs) for b in bases.values()] 146 | [self.check_image("{}-{}".format(b.name, s.name), dirs) 147 | for b in bases.values() 148 | for s in stacks.values()] 149 | 150 | def tearDown(self): 151 | os.chdir('..') 152 | shutil.rmtree('test-stackbuild', ignore_errors=False) 153 | 154 | class TestToolkit3Service(SHToolkitTest): 155 | repo_name = 'registry.stackhut.com:5000/mands/test-service' 156 | image_name = '{}:latest'.format(repo_name) 157 | 158 | @classmethod 159 | def setUpClass(cls): 160 | copy_config('mands') 161 | os.mkdir('test-service') 162 | os.chdir('test-service') 163 | 164 | cls.docker_client = get_docker().client 165 | 166 | # delete any image if exists 167 | try: 168 | cls.docker_client.remove_image(cls.image_name, force=True) 169 | except: 170 | pass 171 | 172 | def test_1_init(self): 173 | out = self.run_toolkit('init', ['debian', 'python'], verbose=True) 174 | # check files copied across 175 | files = ['Hutfile.yaml', 'api.idl', 'README.md', 'app.py'] 176 | [self.assertTrue(os.path.exists(f)) for f in files] 177 | 178 | def test_2_build(self): 179 | out = self.run_toolkit('build', ['--force', '--full', '--dev'], verbose=True) 180 | # check image exists 181 | images = self.docker_client.images(self.repo_name) 182 | self.assertGreater(len(images), 0) 183 | 184 | def test_3_run(self): 185 | # out = self.run_toolkit('run', verbose=True, _bg=True) 186 | out = sh.stackhut('-v', 'run', _out=lambda x:print(x.strip()), _bg=True) 187 | time.sleep(5) 188 | 189 | # use the client lib to send some requests 190 | sh_client = client.SHService('mands', 'test-service', host='http://localhost:6000') 191 | # valid request 192 | res = sh_client.add(1,2) 193 | self.assertEqual(res, 3) 194 | # invalid request 195 | try: 196 | res = sh_client.sub(1,2) 197 | except client.SHRPCError as e: 198 | self.assertEqual(e.code, rpc.ERR_METHOD_NOT_FOUND) 199 | self.assertIn('sub', e.message) 200 | 201 | out.process.signal(2) 202 | out.wait() 203 | self.assertEqual(0, out.exit_code) 204 | 205 | @unittest.skip('Not ready') 206 | def test_4_deploy(self): 207 | out = self.run_toolkit('deploy', ['--local'], verbose=True) 208 | 209 | # test by making a request to live API and checking response 210 | time.sleep(20) # is this needed? 211 | 212 | c = client.SHService('mands', 'test-service') 213 | res = c.add(1,2) 214 | self.assertEqual(res, 3) 215 | 216 | @classmethod 217 | def tearDownClass(cls): 218 | os.chdir('..') 219 | shutil.rmtree('test-service', ignore_errors=False) 220 | cls.docker_client.remove_image(cls.image_name, force=True) 221 | 222 | 223 | if __name__ == '__main__': 224 | unittest.main() 225 | 226 | 227 | # tests to run - commands and high-level func 228 | # login and info 229 | # build base os and stacks 230 | # init project and build, run, (deploy?) 231 | -------------------------------------------------------------------------------- /toolkit.py: -------------------------------------------------------------------------------- 1 | from stackhut_toolkit.__main__ import main 2 | 3 | if __name__ == '__main__': 4 | main() 5 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py34 3 | 4 | [testenv] 5 | #setenv = 6 | # PYTHONPATH = {toxinidir}:{toxinidir}/stackhut 7 | commands = python3 setup.py test 8 | deps = 9 | -r{toxinidir}/requirements.txt 10 | --------------------------------------------------------------------------------