├── .gitignore ├── .travis.yml ├── CHANGES.rst ├── LICENSE ├── Readme.md ├── docs ├── Makefile ├── conf.py └── index.rst ├── firefly ├── __init__.py ├── app.py ├── client.py ├── main.py ├── utils.py ├── validator.py └── version.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── test_app.py ├── test_client.py ├── test_main.py └── test_validator.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__/ 3 | *.egg-info/ 4 | docs/_build/ 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.5" 5 | - "3.6" 6 | install: "pip install -r requirements.txt 'pytest>=3.1.1'" 7 | script: pytest 8 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | Firefly changelog 2 | ================= 3 | 4 | Version 0.1.11 - 2018-01-04 5 | --------------------------- 6 | 7 | * Added better handling of validation errors to the client 8 | * Added better error message when the client function is called with positional arguments instead of keyword arguments 9 | 10 | Version 0.1.10 - 2017-11-13 11 | --------------------------- 12 | 13 | * Fixed the issue that was causing the request local data to leak across requests 14 | 15 | Version 0.1.9 - 2017-09-27 16 | -------------------------- 17 | 18 | * Fixed error invoking the functions from the client when the firefly app is secured using auth token 19 | 20 | Version 0.1.8 - 2017-09-17 21 | -------------------------- 22 | 23 | * Added a simple hack to allow sending custom http status codes from functions 24 | 25 | Version 0.1.7 - 2017-09-16 26 | -------------------------- 27 | 28 | * Added a hack to allow extending firefly 29 | * Made it possible to inject new headers when sending a request by extending the Client 30 | * Better error reporting in the client when the server is not running 31 | 32 | Version 0.1.6 - 2017-09-14 33 | -------------------------- 34 | 35 | * Added support for logging 36 | * Fixed the issue of client not using the path specified in the function specs 37 | 38 | Version 0.1.5 - 2017-08-23 39 | -------------------------- 40 | 41 | * Added support for returning a file object from a function 42 | * Added support for specifying config file as environment variable 43 | * Better error reporting 44 | 45 | Version 0.1.4 - 2017-07-25 46 | -------------------------- 47 | 48 | * Bug fixes 49 | 50 | Version 0.1.3 - 2017-07-25 51 | -------------------------- 52 | 53 | * Fixed the issue with the client when the URL has training / character 54 | * Added support for docstrings in the client functions 55 | * Added support for sending files to the firefly app using multipart/form-data content-type 56 | 57 | Version 0.1.2 - 2017-07-19 58 | -------------------------- 59 | 60 | * Switched to using wsgiref as the default server instead of gunicorn for better portability 61 | * Updated documentation to show how to deploy using gunicorn and on Heroku 62 | * Better error reporting 63 | 64 | Version 0.1.1 - 2017-07-10 65 | -------------------------- 66 | 67 | * Added support for token-based authentication 68 | 69 | Version 0.1.0 - 2017-06-30 70 | -------------------------- 71 | 72 | * First public release 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Firefly 2 | 3 | [![Build Status](https://travis-ci.org/rorodata/firefly.svg?branch=master)](https://travis-ci.org/rorodata/firefly) 4 | 5 | Function as a service. 6 | 7 | # How to install? 8 | 9 | Install firefly from source using: 10 | 11 | pip install firefly-python 12 | 13 | # How to use? 14 | 15 | Create a simple python function. 16 | 17 | # fib.py 18 | 19 | def fib(n): 20 | if n == 0 or n == 1: 21 | return 1 22 | else: 23 | return fib(n-1) + fib(n-2) 24 | 25 | And run it using firefly. 26 | 27 | $ firefly fib.fib 28 | http://127.0.0.1:8000/ 29 | ... 30 | 31 | That started the fib function as a service listening at . 32 | 33 | Let us see how to use it with a client. 34 | 35 | >>> import firefly 36 | >>> client = firefly.Client("http://127.0.0.1:8000/") 37 | >>> client.fib(n=10) 38 | 89 39 | 40 | The service can also be invoked by sending a POST request. 41 | 42 | $ curl -d '{"n": 10}' http://127.0.0.1:8000/fib 43 | 89 44 | 45 | # Documentation 46 | 47 | 48 | 49 | # Features Planned 50 | 51 | * Auto reload 52 | * supporting other input and output content-types in addition to json. (for example, a function to resize an image) 53 | * serverless deployment 54 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = Firefly 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Firefly documentation build configuration file, created by 5 | # sphinx-quickstart on Wed Jun 21 11:32:55 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = [] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix(es) of source filenames. 40 | # You can specify multiple suffix as a list of string: 41 | # 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = 'Firefly' 50 | copyright = '2017, rorodata' 51 | author = 'rorodata' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This patterns also effect to html_static_path and html_extra_path 72 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = 'sphinx' 76 | 77 | # If true, `todo` and `todoList` produce output, else they produce nothing. 78 | todo_include_todos = False 79 | 80 | 81 | # -- Options for HTML output ---------------------------------------------- 82 | 83 | # The theme to use for HTML and HTML Help pages. See the documentation for 84 | # a list of builtin themes. 85 | # 86 | html_theme = 'alabaster' 87 | 88 | # Theme options are theme-specific and customize the look and feel of a theme 89 | # further. For a list of options available for each theme, see the 90 | # documentation. 91 | # 92 | # html_theme_options = {} 93 | 94 | # Add any paths that contain custom static files (such as style sheets) here, 95 | # relative to this directory. They are copied after the builtin static files, 96 | # so a file named "default.css" will overwrite the builtin "default.css". 97 | html_static_path = ['_static'] 98 | 99 | 100 | # -- Options for HTMLHelp output ------------------------------------------ 101 | 102 | # Output file base name for HTML help builder. 103 | htmlhelp_basename = 'Fireflydoc' 104 | 105 | 106 | # -- Options for LaTeX output --------------------------------------------- 107 | 108 | latex_elements = { 109 | # The paper size ('letterpaper' or 'a4paper'). 110 | # 111 | # 'papersize': 'letterpaper', 112 | 113 | # The font size ('10pt', '11pt' or '12pt'). 114 | # 115 | # 'pointsize': '10pt', 116 | 117 | # Additional stuff for the LaTeX preamble. 118 | # 119 | # 'preamble': '', 120 | 121 | # Latex figure (float) alignment 122 | # 123 | # 'figure_align': 'htbp', 124 | } 125 | 126 | # Grouping the document tree into LaTeX files. List of tuples 127 | # (source start file, target name, title, 128 | # author, documentclass [howto, manual, or own class]). 129 | latex_documents = [ 130 | (master_doc, 'Firefly.tex', 'Firefly Documentation', 131 | 'rorodata', 'manual'), 132 | ] 133 | 134 | 135 | # -- Options for manual page output --------------------------------------- 136 | 137 | # One entry per manual page. List of tuples 138 | # (source start file, name, description, authors, manual section). 139 | man_pages = [ 140 | (master_doc, 'firefly', 'Firefly Documentation', 141 | [author], 1) 142 | ] 143 | 144 | 145 | # -- Options for Texinfo output ------------------------------------------- 146 | 147 | # Grouping the document tree into Texinfo files. List of tuples 148 | # (source start file, target name, title, author, 149 | # dir menu entry, description, category) 150 | texinfo_documents = [ 151 | (master_doc, 'Firefly', 'Firefly Documentation', 152 | author, 'Firefly', 'One line description of project.', 153 | 'Miscellaneous'), 154 | ] 155 | 156 | 157 | 158 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Firefly documentation master file, created by 2 | sphinx-quickstart on Wed Jun 21 11:32:55 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | firefly 7 | ======= 8 | 9 | firefly is a function as a service framework which can be used to deploy functions as a web service. In turn, the functions can be accessed over a REST based API. It works like RPC, but it also provides a way to customize the URLs to allow great RESTful API as well. 10 | 11 | Firefly was initially started to make deploying machine learning models easier, but it can used for other use cases equally well. 12 | 13 | Installation 14 | ------------ 15 | 16 | ``firefly`` can be installed by using ``pip`` as: 17 | :: 18 | 19 | $ pip install firefly-python 20 | 21 | You can check the installation by using: 22 | :: 23 | 24 | $ firefly -h 25 | 26 | Basic Usage 27 | ----------- 28 | 29 | Create a simple python function: 30 | :: 31 | 32 | # funcs.py 33 | 34 | def square(n): 35 | return n**2 36 | 37 | And then this function can run through ``firefly`` by the following: 38 | :: 39 | 40 | $ firefly funcs.square 41 | http://127.0.0.1:8000/ 42 | ... 43 | 44 | This function is now accessible at ``http://127.0.0.1:8000/square`` . 45 | An inbuilt ``Client`` is also provided to communicate with the ``firefly`` 46 | server. Example usage of the client: 47 | :: 48 | 49 | >>> import firefly 50 | >>> client = firefly.Client("http://127.0.0.1:8000/") 51 | >>> client.square(n=4) 52 | 16 53 | 54 | Besides that, you can also use ``curl`` or any software through which you can do 55 | a POST request to the endpoint. 56 | :: 57 | 58 | $ curl -d '{"n": 4}' http://127.0.0.1:8000/square 59 | 16 60 | 61 | ``firefly`` supports for any number of functions. You can pass multiple 62 | functions as: 63 | :: 64 | 65 | $ firefly funcs.square funcs.cube 66 | 67 | The functions ``square`` and ``cube`` can be accessed at ``127.0.0.1:8000/square`` 68 | and ``127.0.0.01:8000/cube`` respectively. 69 | 70 | Authentication 71 | -------------- 72 | 73 | ``firefly`` also supports token-based authentication. You will need to pass a token 74 | through the CLI or the config file. 75 | :: 76 | 77 | $ # CLI Usage 78 | $ firefly --token abcd1234 funcs.square 79 | http://127.0.0.1:8000/ 80 | 81 | 82 | The token now needs to be passed with each request. 83 | :: 84 | 85 | >>> import firefly 86 | >>> client = firefly.Client("http://127.0.0.1:8000/", auth_token="abcd1234") 87 | >>> client.square(n=4) 88 | 16 89 | 90 | If you are using anything other than inbuilt-client, the ``Authorization`` 91 | HTTP header needs to be set in the POST request. 92 | :: 93 | 94 | $ curl -d '{"n": 4}' -H "Authorization: Token abcd1234" http://127.0.0.1:8000/square 95 | 16 96 | 97 | Using a config file 98 | ------------------- 99 | 100 | ``firefly`` can also take a configuration file with the following schema: 101 | :: 102 | 103 | # config.yml 104 | 105 | version: 1.0 106 | token: "abcd1234" 107 | functions: 108 | square: 109 | path: "/square" 110 | function: "funcs.square" 111 | cube: 112 | path: "/cube" 113 | function: "funcs.cube" 114 | ... 115 | 116 | You can specify the configuration file as: 117 | :: 118 | 119 | $ firefly -c config.yml 120 | http://127.0.0.1:8000/ 121 | ... 122 | 123 | Deploying a ML model 124 | -------------------- 125 | 126 | Machine Learning models can also be deployed by using ``firefly``. You need to 127 | wrap the prediction logic as a function. For example, if you have a directory 128 | as follows: 129 | :: 130 | 131 | $ ls 132 | model.py classifier.pkl 133 | 134 | where ``classifier.pkl`` is a ``joblib`` dump of a SVM Classifier model. 135 | :: 136 | 137 | # model.py 138 | from sklearn.externals import joblib 139 | 140 | clf = joblib.load('classifier.pkl') 141 | 142 | def predict(a): 143 | predicted = clf.predict(a) # predicted is 1x1 numpy array 144 | return int(predicted[0]) 145 | 146 | Invoke ``firefly`` as: 147 | :: 148 | 149 | $ firefly model.predict 150 | http://127.0.0.1:8000/ 151 | ... 152 | 153 | Now, you can access this by: 154 | :: 155 | 156 | >>> import firefly 157 | >>> client = firefly.Client("http://127.0.0.1:8000/") 158 | >>> client.predict(a=[5, 8]) 159 | 1 160 | 161 | You can use any model provided the function returns a JSON friendly data type. 162 | 163 | Firefly with gunicorn 164 | --------------------- 165 | 166 | ``firefly`` applications can also be deployed using `gunicorn `_ . 167 | The arguments that are passed to ``firefly`` via CLI can be set as environment 168 | variables. 169 | :: 170 | 171 | $ gunicorn --preload firefly.main:app -e FIREFLY_FUNCTIONS="funcs.square" -e FIREFLY_TOKEN="abcd1234" 172 | [2017-07-19 14:47:57 +0530] [29601] [INFO] Starting gunicorn 19.7.1 173 | [2017-07-19 14:47:57 +0530] [29601] [INFO] Listening at: http://127.0.0.1:8000 (29601) 174 | [2017-07-19 14:47:57 +0530] [29601] [INFO] Using worker: sync 175 | [2017-07-19 14:47:57 +0530] [29604] [INFO] Booting worker with pid: 29604 176 | 177 | If you want to deploy multiple functions, pass them as a comma-seperated list. 178 | :: 179 | 180 | $ gunicorn --preload firefly.main.app -e FIREFLY_FUNCTIONS="funcs.square,funcs.cube" -e FIREFLY_TOKEN="abcd1234" 181 | 182 | Deployment on Heroku 183 | -------------------- 184 | 185 | ``firefly`` functions are deploying on any cloud platform. This section shows 186 | how you can deploy ML models to `Heroku `_ . There are two 187 | important files apart from your model code that you will need to have in your 188 | application root directory - ``Procfile`` and ``requirements.txt``. ``Procfile`` 189 | lets Heroku know what sort of process you want to run and what command it should 190 | run. ``requirements.txt`` specifies dependencies of your code. 191 | :: 192 | 193 | # requirements.txt 194 | firefly-python 195 | sklearn 196 | numpy 197 | scipy 198 | 199 | This ``Procfile`` tells Heroku to run ``firefly`` serving the ``predict`` 200 | function inside the ``model`` script. 201 | :: 202 | 203 | # Procfile 204 | web: gunicorn --preload firefly.main:app -e FIREFLY_FUNCTIONS="model.predict" 205 | 206 | :: 207 | 208 | $ ls 209 | model.py classifier.pkl requirements.txt Procfile 210 | 211 | Now that everything is setup on your machine, we can deploy the application to 212 | Heroku. 213 | 214 | :: 215 | 216 | $ git add . 217 | 218 | $ git commit -m "Added a Procfile." 219 | 220 | $ heroku login 221 | Enter your Heroku credentials. 222 | ... 223 | 224 | $ heroku create 225 | Creating intense-falls-9163... done, stack is cedar 226 | http://intense-falls-9163.herokuapp.com/ | git@heroku.com:intense-falls-9163.git 227 | Git remote heroku added 228 | 229 | $ git push heroku master 230 | ... 231 | -----> Python app detected 232 | ... 233 | -----> Launching... done, v7 234 | https://intense-falls-9163.herokuapp.com/ deployed to Heroku 235 | 236 | For more information about deploying python applications to Heroku, go 237 | `here `_ . 238 | -------------------------------------------------------------------------------- /firefly/__init__.py: -------------------------------------------------------------------------------- 1 | from .app import Firefly 2 | from .client import Client 3 | from .version import __version__ 4 | -------------------------------------------------------------------------------- /firefly/app.py: -------------------------------------------------------------------------------- 1 | import cgi 2 | from webob import Request, Response 3 | from webob.exc import HTTPNotFound 4 | import json 5 | import functools 6 | import logging 7 | from .validator import validate_args, ValidationError 8 | from .utils import json_encode, is_file, FileIter 9 | from .version import __version__ 10 | import threading 11 | from wsgiref.simple_server import make_server 12 | 13 | try: 14 | from inspect import signature, _empty 15 | except: 16 | from funcsigs import signature, _empty 17 | 18 | logger = logging.getLogger("firefly") 19 | 20 | # XXX-Anand 21 | # Hack to store the request-local context. 22 | # Need to think of a better way to handle this 23 | # or switch to Flask. 24 | ctx = threading.local() 25 | ctx.request = None 26 | 27 | class Firefly(object): 28 | def __init__(self, auth_token=None, allowed_origins=""): 29 | """Creates a firefly application. 30 | 31 | If the optional parameter auth_token is specified, the 32 | only the requests which provide that auth token in authorization 33 | header are allowed. 34 | 35 | The Cross Origin Request Sharing is disabled by default. To enable it, 36 | pass the allowed origins as allowed_origins. To allow all origins, set it 37 | to ``*``. 38 | 39 | :param auth_token: the auto_token for the application 40 | :param allowed_origins: allowed origins for cross-origin requests 41 | """ 42 | self.mapping = {} 43 | self.add_route('/', self.generate_index,internal=True) 44 | self.auth_token = auth_token 45 | self.allowed_origins = allowed_origins 46 | 47 | 48 | def set_auth_token(self, token): 49 | self.auth_token = token 50 | 51 | def set_allowed_origins(self, allowed_origins): 52 | # Also support mutliple origins as a list. 53 | if isinstance(allowed_origins, list): 54 | allowed_origins = ", ".join(allowed_origins) 55 | self.allowed_origins = allowed_origins or "" 56 | 57 | def function(self, func=None, name=None, path=None): 58 | if func is None: 59 | return functools.partial(self.function, name=name, path=path) 60 | name = name or func.__name__ 61 | path = path or "/" + name 62 | self.add_route(path=path, function=func, function_name=name) 63 | return func 64 | 65 | def add_route(self, path, function, function_name=None, **kwargs): 66 | self.mapping[path] = FireflyFunction(function, function_name, **kwargs) 67 | 68 | def generate_function_list(self): 69 | return {f.name: {"path": path, "doc": f.doc, "parameters": f.sig} 70 | for path, f in self.mapping.items() 71 | if f.options.get("internal") != True} 72 | 73 | def generate_index(self): 74 | help_dict = { 75 | "app": "firefly", 76 | "version": __version__, 77 | "functions": self.generate_function_list() 78 | } 79 | return help_dict 80 | 81 | def __call__(self, environ, start_response): 82 | request = Request(environ) 83 | response = self.process_request(request) 84 | return response(environ, start_response) 85 | 86 | def verify_auth_token(self, request): 87 | return not self.auth_token or self.auth_token == self._get_auth_token(request) 88 | 89 | def _get_auth_token(self, request): 90 | auth = request.headers.get("Authorization") 91 | if auth and auth.lower().startswith("token"): 92 | return auth[len("token"):].strip() 93 | 94 | def http_error(self, status, error=None): 95 | response = Response() 96 | response.status = status 97 | response.text = json_encode({"error": error}) 98 | return response 99 | 100 | def _prepare_cors_headers(self): 101 | if self.allowed_origins: 102 | headers = { 103 | 'Access-Control-Allow-Origin': self.allowed_origins, 104 | 'Access-Control-Allow-Methods': 'GET,POST', 105 | 'Access-Control-Allow-Headers': 'Content-Type, Authorization' 106 | } 107 | return list(headers.items()) 108 | else: 109 | return [] 110 | 111 | def process_request(self, request): 112 | if request.method == 'OPTIONS': 113 | response = Response(status='200 OK', body=b'') 114 | response.headerlist += self._prepare_cors_headers() 115 | return response 116 | 117 | if not self.verify_auth_token(request): 118 | return self.http_error('403 Forbidden', error='Invalid auth token') 119 | 120 | # Clear all the existing state, if any 121 | ctx.__dict__.clear() 122 | 123 | ctx.request = request 124 | 125 | path = request.path_info 126 | if path in self.mapping: 127 | func = self.mapping[path] 128 | response = func(request) 129 | response.headerlist += self._prepare_cors_headers() 130 | else: 131 | response = self.http_error('404 Not Found', error="Not found: " + path) 132 | 133 | ctx.request = None 134 | return response 135 | 136 | def run(self, host=None, port=None): 137 | host = host or "localhost" 138 | port = port or 8000 139 | print("http://{}:{}/".format(host, port)) 140 | server = make_server(host, port, self) 141 | server.serve_forever() 142 | 143 | class FireflyFunction(object): 144 | def __init__(self, function, function_name=None, **options): 145 | self.function = function 146 | self.options = options 147 | self.name = function_name or function.__name__ 148 | self.doc = function.__doc__ or "" 149 | self.sig = self.generate_signature(function) 150 | 151 | def __repr__(self): 152 | return "" % self.function 153 | 154 | def __call__(self, request): 155 | if self.options.get("internal", False): 156 | return self.make_response(self.function()) 157 | 158 | logger.info("calling function %s", self.name) 159 | try: 160 | kwargs = self.get_inputs(request) 161 | except ValueError as err: 162 | logger.warn("Function %s failed with ValueError: %s.", self.name, err) 163 | return self.make_response({"error": str(err)}, status=400) 164 | 165 | try: 166 | validate_args(self.function, kwargs) 167 | except ValidationError as err: 168 | logger.warn("Function %s failed with ValidationError: %s.", self.name, err) 169 | return self.make_response({"error": str(err)}, status=422) 170 | 171 | try: 172 | result = self.function(**kwargs) 173 | except HTTPError as e: 174 | return e.get_response() 175 | except Exception as err: 176 | logger.error("Function %s failed with exception.", self.name, exc_info=True) 177 | return self.make_response( 178 | {"error": "{}: {}".format(err.__class__.__name__, str(err))}, status=500 179 | ) 180 | return self.make_response(result) 181 | 182 | def get_inputs(self, request): 183 | content_type = self.get_content_type(request) 184 | if content_type == 'multipart/form-data': 185 | return self.get_multipart_formdata_inputs(request) 186 | else: 187 | return json.loads(request.body.decode('utf-8')) 188 | 189 | def get_content_type(self, request): 190 | content_type = request.headers.get('Content-Type', 'application/octet-stream') 191 | return content_type.split(';')[0] 192 | 193 | def get_multipart_formdata_inputs(self, request): 194 | d = {} 195 | for name, value in request.POST.items(): 196 | if isinstance(value, cgi.FieldStorage): 197 | value = value.file 198 | d[name] = value 199 | return d 200 | 201 | def make_response(self, result, status=200): 202 | if is_file(result): 203 | response = Response(content_type='application/octet-stream') 204 | response.app_iter = FileIter(result) 205 | else: 206 | response = Response(content_type='application/json', 207 | charset='utf-8') 208 | response.text = json_encode(result) 209 | response.status = status 210 | return response 211 | 212 | def generate_signature(self, f): 213 | func_sig = signature(f) 214 | params = [] 215 | 216 | for param_name, param_obj in func_sig.parameters.items(): 217 | param = { 218 | "name": param_name, 219 | "kind": str(param_obj.kind) 220 | } 221 | if param_obj.default is not _empty: 222 | param["default"] = param_obj.default 223 | params += [param] 224 | 225 | return params 226 | 227 | class HTTPError(Exception): 228 | """Exception to be raised to send different HTTP status codes. 229 | """ 230 | def __init__(self, status_code, body, headers={}): 231 | self.status_code = status_code 232 | self.body = body 233 | self.headers = headers 234 | 235 | def get_response(self): 236 | response = Response() 237 | response.status = self.status_code 238 | response.text = self.body 239 | response.headers.update(self.headers) 240 | return response 241 | -------------------------------------------------------------------------------- /firefly/client.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from requests import ConnectionError 3 | from .validator import ValidationError 4 | import logging 5 | import time 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | class Client: 10 | def __init__(self, server_url, auth_token=None): 11 | # strip trailing / to avoid double / chars in the URL 12 | self.server_url = server_url.rstrip("/") 13 | self.auth_token = auth_token 14 | self._metadata = None 15 | 16 | def __getattr__(self, func_name): 17 | return RemoteFunction(self, func_name) 18 | 19 | def call_func(self, func_name, **kwargs): 20 | path = self._get_path(func_name) 21 | return self.request(path, **kwargs) 22 | 23 | def request(self, _path, **kwargs): 24 | url = self.server_url + _path 25 | t0 = time.time() 26 | try: 27 | headers = self.prepare_headers() 28 | data, files = self.decouple_files(kwargs) 29 | if files: 30 | response = requests.post(url, data=data, files=files, headers=headers, stream=True) 31 | else: 32 | response = requests.post(url, json=data, headers=headers, stream=True) 33 | except ConnectionError: 34 | raise FireflyError('Unable to connect to the server, please try again later.') 35 | finally: 36 | t1 = time.time() 37 | logger.info("%0.3f: POST %s", t1-t0, url) 38 | return self.handle_response(response) 39 | 40 | def prepare_headers(self): 41 | """Prepares headers for sending a request to the firefly server. 42 | """ 43 | headers = {} 44 | if self.auth_token: 45 | headers['Authorization'] = 'Token {}'.format(self.auth_token) 46 | return headers 47 | 48 | 49 | def _get_path(self, func_name): 50 | functions = self._metadata.get('functions', {}) 51 | func_info = functions.get(func_name) or {"path": "/" + func_name} 52 | return func_info["path"] 53 | 54 | def _get_metadata(self): 55 | try: 56 | if self._metadata is None: 57 | url = self.server_url + "/" 58 | headers = self.prepare_headers() 59 | response = requests.get(url, headers=headers) 60 | 61 | if response.status_code == 200: 62 | self._metadata = response.json() 63 | else: 64 | raise FireflyError( 65 | "Failed to contact the server (http status code {}).".format( 66 | response.status_code)) 67 | return self._metadata 68 | except ConnectionError as err: 69 | raise FireflyError('Unable to connect to the server, please try again later.') 70 | 71 | def get_doc(self, func_name): 72 | metadata = self._get_metadata().get("functions", {}) 73 | return metadata.get(func_name, {}).get("doc") or "" 74 | 75 | def decouple_files(self, kwargs): 76 | data = {arg: value for arg, value in kwargs.items() if not self.is_file(value)} 77 | files = {arg: value for arg, value in kwargs.items() if self.is_file(value)} 78 | return data, files 79 | 80 | def is_file(self, value): 81 | return hasattr(value, 'read') or hasattr(value, 'readlines') 82 | 83 | def handle_response(self, response): 84 | if response.status_code == 200: 85 | return self.decode_response(response) 86 | elif response.status_code == 400: 87 | try: 88 | error = response.json()["error"] 89 | except (KeyError, ValueError): 90 | error = "Bad Request" 91 | raise ValueError(error) 92 | elif response.status_code == 403: 93 | raise FireflyError("Authorization token mismatch.") 94 | elif response.status_code == 404: 95 | raise FireflyError("Requested function not found") 96 | elif response.status_code == 422: 97 | raise ValidationError(response.json()["error"]) 98 | elif response.status_code == 500: 99 | if response.headers["Content-Type"] == "application/json": 100 | raise FireflyError(response.json()["error"]) 101 | else: 102 | raise FireflyError(response.text) 103 | else: 104 | raise FireflyError("Oops! Something really bad happened") 105 | 106 | def decode_response(self, response): 107 | if response.headers["Content-Type"] == "application/octet-stream": 108 | return response.raw 109 | else: 110 | return response.json() 111 | 112 | def RemoteFunction(client, func_name): 113 | def wrapped(*args, **kwargs): 114 | if args: 115 | raise FireflyError('Firefly functions only accept named arguments') 116 | return client.call_func(func_name, **kwargs) 117 | wrapped.__name__ = func_name 118 | wrapped.__qualname__ = func_name 119 | wrapped.__doc__ = client.get_doc(func_name) 120 | return wrapped 121 | 122 | class FireflyError(Exception): 123 | pass 124 | -------------------------------------------------------------------------------- /firefly/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import argparse 4 | import importlib 5 | import yaml 6 | import logging 7 | from .app import Firefly 8 | from .validator import ValidationError, FireflyError 9 | from .version import __version__ 10 | from wsgiref.simple_server import make_server 11 | 12 | logger = logging.getLogger("firefly") 13 | 14 | def load_from_env(): 15 | functions = None 16 | token = None 17 | allow_origins = '' 18 | 19 | if 'FIREFLY_FUNCTIONS' in os.environ: 20 | function_names = os.environ['FIREFLY_FUNCTIONS'].split(",") 21 | try: 22 | functions = load_functions(function_names) 23 | except (ImportError, AttributeError) as err: 24 | sys.exit(1) 25 | 26 | if 'FIREFLY_TOKEN' in os.environ: 27 | token = os.environ['FIREFLY_TOKEN'] 28 | 29 | if 'FIREFLY_ALLOW_ORIGINS' in os.environ: 30 | allow_origins = os.environ['FIREFLY_ALLOW_ORIGINS'] 31 | 32 | if 'FIREFLY_CONFIG' in os.environ: 33 | logger.info("loading config file: %s", os.environ['FIREFLY_CONFIG']) 34 | functions, token = parse_config_data(parse_config_file(os.environ['FIREFLY_CONFIG'])) 35 | 36 | if functions: 37 | add_routes(app, functions) 38 | 39 | if token: 40 | app.set_auth_token(token) 41 | 42 | if allow_origins: 43 | app.set_allowed_origins(allow_origins) 44 | 45 | def parse_args(): 46 | p = argparse.ArgumentParser() 47 | p.add_argument("--version", action="store_true", help="Prints the firefly version") 48 | p.add_argument("-t", "--token", help="token to authenticate the requests") 49 | p.add_argument("-b", "--bind", dest="ADDRESS", default="127.0.0.1:8000") 50 | p.add_argument("-c", "--config", dest="config_file", default=None) 51 | p.add_argument("--allow-origins", default=None, help="Origins to allow for cross-origin resource sharing") 52 | p.add_argument("functions", nargs='*', help="functions to serve") 53 | return p.parse_args() 54 | 55 | def load_function(function_spec, path=None, name=None): 56 | if "." not in function_spec: 57 | raise Exception("Invalid function: {}, please specify it as module.function".format(function_spec)) 58 | 59 | mod_name, func_name = function_spec.rsplit(".", 1) 60 | try: 61 | mod = importlib.import_module(mod_name) 62 | func = getattr(mod, func_name) 63 | except (ImportError, AttributeError) as err: 64 | print("Failed to load {}: {}".format(function_spec, str(err))) 65 | raise 66 | path = path or "/"+func_name 67 | name = name or func_name 68 | return (path, name, func) 69 | 70 | def load_functions(function_specs): 71 | return [load_function(function_spec) for function_spec in function_specs] 72 | 73 | def parse_config_file(config_file): 74 | if not os.path.exists(config_file): 75 | raise FireflyError("Specified config file does not exist.") 76 | with open(config_file) as f: 77 | config_dict = yaml.safe_load(f) 78 | return config_dict 79 | 80 | def parse_config_data(config_dict): 81 | functions = [(load_function(f["function"], path=f.get("path"), name=name, )) 82 | for name, f in config_dict["functions"].items()] 83 | token = config_dict.get("token", None) 84 | return functions, token 85 | 86 | def add_routes(app, functions): 87 | for path, name, function in functions: 88 | app.add_route(path, function, name) 89 | 90 | def setup_logger(): 91 | level = logging.INFO 92 | logging.basicConfig( 93 | level=level, 94 | format="%(asctime)s %(name)s [%(levelname)s] %(message)s", 95 | datefmt='%Y-%m-%d %H:%M:%S') 96 | 97 | def main(): 98 | # ensure current directory is added to sys.path 99 | if "" not in sys.path: 100 | sys.path.insert(0, "") 101 | 102 | args = parse_args() 103 | 104 | if args.version: 105 | print("Firefly Version {}".format(__version__)) 106 | return 107 | 108 | if (args.functions and args.config_file) or (not args.functions and not args.config_file): 109 | raise FireflyError("Invalid arguments provided. Please specify either a config file or a list of functions.") 110 | 111 | token = None 112 | 113 | if len(args.functions): 114 | functions = load_functions(args.functions) 115 | elif args.config_file: 116 | functions, token = parse_config_data(parse_config_file(args.config_file)) 117 | 118 | token = token or args.token 119 | 120 | app.set_auth_token(token) 121 | if args.allow_origins: 122 | app.set_allowed_origins(args.allow_origins) 123 | add_routes(app, functions) 124 | 125 | host, port = args.ADDRESS.split(":", 1) 126 | port = int(port) 127 | print("http://{}/".format(args.ADDRESS)) 128 | server = make_server(host, port, app) 129 | server.serve_forever() 130 | 131 | setup_logger() 132 | logger.info("Starting Firefly...") 133 | app = Firefly() 134 | load_from_env() 135 | -------------------------------------------------------------------------------- /firefly/utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | 4 | PY2 = (sys.version_info.major == 2) 5 | PY3 = (sys.version_info.major == 3) 6 | 7 | def json_encode(data): 8 | result = json.dumps(data) 9 | if PY2: 10 | result = result.decode('utf-8') 11 | return result 12 | 13 | def is_file(obj): 14 | return hasattr(obj, "read") 15 | 16 | class FileIter: 17 | def __init__(self, fileobj, chunk_size=4096): 18 | self.fileobj = fileobj 19 | self.chunk_size = chunk_size 20 | 21 | def __iter__(self): 22 | while True: 23 | chunk = self.fileobj.read(self.chunk_size) 24 | if not chunk: 25 | break 26 | yield chunk 27 | -------------------------------------------------------------------------------- /firefly/validator.py: -------------------------------------------------------------------------------- 1 | from .utils import PY2, PY3 2 | 3 | if PY3: 4 | from inspect import signature 5 | else: 6 | from funcsigs import signature 7 | 8 | class ValidationError(Exception): 9 | pass 10 | 11 | class FireflyError(Exception): 12 | pass 13 | 14 | def validate_args(function, kwargs): 15 | function_signature = signature(function) 16 | try: 17 | function_signature.bind(**kwargs) 18 | except TypeError as err: 19 | raise ValidationError(str(err)) 20 | -------------------------------------------------------------------------------- /firefly/version.py: -------------------------------------------------------------------------------- 1 | 2 | __version__ = "0.1.15" 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | gunicorn>=19.7.1 2 | WebOb>=1.7.2 3 | requests>=2.18.1 4 | PyYAML>=3.12 5 | funcsigs>=1.0.2 ; python_version < '3' 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Firefly 3 | ------- 4 | 5 | Firefly is a tool to expose Python functions as RESTful APIs. 6 | 7 | Install 8 | ~~~~~~~ 9 | 10 | It can be installed using pip. 11 | 12 | ..code:: bash 13 | 14 | $ pip install firefly-python 15 | 16 | Usage 17 | ~~~~~ 18 | 19 | Write a python function: 20 | 21 | ..code:: python 22 | 23 | # sq.py 24 | def square(n): 25 | return n*n 26 | 27 | And run it with firefly: 28 | 29 | ..code:: bash 30 | 31 | $ firefly sq.square 32 | [2017-06-08 12:45:11 +0530] [20237] [INFO] Starting gunicorn 19.7.1 33 | [2017-06-08 12:45:11 +0530] [20237] [INFO] Listening at: http://127.0.0.1:8000 (20237) 34 | ... 35 | 36 | Firefly provides a simple client interface to interact with the server. 37 | 38 | ..code:: python 39 | 40 | >>> from firefly.client import Client 41 | >>> client = Client("http://127.0.0.1:8000") 42 | >>> client.square(n=4) 43 | 16 44 | 45 | Or, you can use the API directly: 46 | 47 | ..code:: bash 48 | 49 | $ curl -d '{"n": 4}' http://127.0.0.1:8000/square 50 | 16 51 | 52 | Links 53 | ~~~~~ 54 | 55 | * `Documentation `_ 56 | * `Github `_ 57 | """ 58 | 59 | from setuptools import setup, find_packages 60 | import os.path 61 | import sys 62 | 63 | PY2 = (sys.version_info.major == 2) 64 | 65 | def get_version(): 66 | """Returns the package version taken from version.py. 67 | """ 68 | root = os.path.dirname(__file__) 69 | version_path = os.path.join(root, "firefly/version.py") 70 | with open(version_path) as f: 71 | code = f.read() 72 | env = {} 73 | exec(code, env, env) 74 | return env['__version__'] 75 | 76 | install_requires = [ 77 | 'gunicorn>=19.7.1', 78 | 'WebOb>=1.7.2', 79 | 'requests>=2.18.1', 80 | 'PyYAML>=3.12' 81 | ] 82 | 83 | if PY2: 84 | install_requires.append('funcsigs>=1.0.2') 85 | 86 | __version__ = get_version() 87 | 88 | setup( 89 | name='firefly-python', 90 | version=__version__, 91 | author='rorodata', 92 | author_email='rorodata.team@gmail.com', 93 | description='deploying functions made easy', 94 | long_description=__doc__, 95 | packages=find_packages(), 96 | include_package_data=True, 97 | install_requires=install_requires, 98 | entry_points=''' 99 | [console_scripts] 100 | firefly=firefly.main:main 101 | ''', 102 | ) 103 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rorodata/firefly/07b76e54cc38c0452dce6dc7faa3c4bda067e20b/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_app.py: -------------------------------------------------------------------------------- 1 | import io 2 | import sys 3 | import pytest 4 | from webob import Request, Response 5 | from firefly.app import Firefly, FireflyFunction, ctx 6 | 7 | py2_only = pytest.mark.skipif(sys.version_info.major >= 3, reason="Requires Python 2") 8 | py3_only = pytest.mark.skipif(sys.version_info.major < 3, reason="Requires Python 3+") 9 | 10 | def square(a): 11 | '''Computes square''' 12 | return a**2 13 | 14 | def dummy(): 15 | return 16 | 17 | class TestFirefly: 18 | def test_generate_function_list(self): 19 | firefly = Firefly() 20 | assert firefly.generate_function_list() == {} 21 | 22 | firefly.add_route("/square", square, "square") 23 | returned_dict = { 24 | "square": { 25 | "path": "/square", 26 | "doc": "Computes square", 27 | "parameters": [ 28 | { 29 | "name": "a", 30 | "kind": "POSITIONAL_OR_KEYWORD" 31 | } 32 | ] 33 | } 34 | } 35 | assert firefly.generate_function_list() == returned_dict 36 | 37 | def test_generate_function_list_for_func_name(self): 38 | firefly = Firefly() 39 | firefly.add_route("/sq2", square, "sq") 40 | returned_dict = { 41 | "sq": { 42 | "path": "/sq2", 43 | "doc": "Computes square", 44 | "parameters": [ 45 | { 46 | "name": "a", 47 | "kind": "POSITIONAL_OR_KEYWORD" 48 | } 49 | ] 50 | } 51 | } 52 | assert firefly.generate_function_list() == returned_dict 53 | 54 | def test_function_call(self): 55 | app = Firefly() 56 | app.add_route("/", square) 57 | 58 | request = Request.blank("/", POST='{"a": 3}') 59 | response = app.process_request(request) 60 | assert response.status == '200 OK' 61 | assert response.text == '9' 62 | 63 | def test_auth_failure(self): 64 | app = Firefly(auth_token='abcd') 65 | app.add_route("/", square) 66 | 67 | request = Request.blank("/", POST='{"a": 3}') 68 | response = app.process_request(request) 69 | print(response.text) 70 | assert response.status == '403 Forbidden' 71 | 72 | headers = { 73 | "Authorization": "token bad-token" 74 | } 75 | request = Request.blank("/", POST='{"a": 3}', headers=headers) 76 | response = app.process_request(request) 77 | assert response.status == '403 Forbidden' 78 | 79 | def test_http_error_404(self): 80 | app = Firefly() 81 | app.add_route("/", square) 82 | 83 | request = Request.blank("/sq", POST='{"a": 3}') 84 | response = app.process_request(request) 85 | assert response.status == '404 Not Found' 86 | 87 | def test_ctx(self): 88 | def peek_ctx(): 89 | keys = sorted(ctx.__dict__.keys()) 90 | return list(keys) 91 | 92 | app = Firefly() 93 | app.add_route("/", peek_ctx) 94 | 95 | request = Request.blank("/", POST='{}') 96 | response = app.process_request(request) 97 | assert response.status == '200 OK' 98 | assert response.json == ['request'] 99 | 100 | def test_ctx_cross_request(self): 101 | def peek_ctx(): 102 | print("peek_ctx", ctx.__dict__) 103 | ctx.count = getattr(ctx, "count", 0) + 1 104 | return ctx.count 105 | 106 | app = Firefly() 107 | app.add_route("/", peek_ctx) 108 | 109 | request = Request.blank("/", POST='{}') 110 | response = app.process_request(request) 111 | assert response.status == '200 OK' 112 | assert response.json == 1 113 | 114 | # Subsequent requests should not have count in the context 115 | request = Request.blank("/", POST='{}') 116 | response = app.process_request(request) 117 | assert response.status == '200 OK' 118 | assert response.json == 1 119 | 120 | class TestFireflyFunction: 121 | def test_call(self): 122 | func = FireflyFunction(square) 123 | request = Request.blank("/square", POST='{"a": 3}') 124 | response = func(request) 125 | assert response.status == '200 OK' 126 | assert response.text == '9' 127 | 128 | def test_call_for_bad_request(self): 129 | def sum(a): 130 | return sum(a) 131 | func = FireflyFunction(sum) 132 | request = Request.blank("/sum", POST='{"a": [3 8]}') 133 | response = func(request) 134 | assert response.status == '400 Bad Request' 135 | 136 | def test_call_for_internal_function_error(self): 137 | def dummy(a): 138 | raise ValueError("This is a test") 139 | req = Request.blank('/dummy', POST='{"a": 1}') 140 | func = FireflyFunction(dummy) 141 | resp = func(req) 142 | assert resp.status == '500 Internal Server Error' 143 | assert resp.json == {'error': 'ValueError: This is a test'} 144 | 145 | def test_call_for_file_inputs(self): 146 | def filesize(data): 147 | return len(data.read()) 148 | f = io.StringIO(u"test file contents") 149 | req = Request.blank('/filesize', POST={'data': ('test', f)}) 150 | func = FireflyFunction(filesize) 151 | resp = func(req) 152 | assert resp.status == '200 OK' 153 | assert resp.body == b'18' 154 | 155 | def test_get_multipart_formdata_inputs_with_files(self): 156 | f = io.StringIO(u"test file contents") 157 | g = io.StringIO(u"test file contents") 158 | req = Request.blank('/filesize', POST={'data': ('test', f)}) 159 | func = FireflyFunction(dummy) 160 | d = func.get_multipart_formdata_inputs(req) 161 | assert d['data'].read().decode() == g.read() 162 | 163 | def test_get_multipart_formdata_inputs_with_combined_inputs(self): 164 | f = io.StringIO(u"test file contents") 165 | g = io.StringIO(u"test file contents") 166 | req = Request.blank('/filesize', POST={'data': ('test', f), 'abc': 'hi', 'xyz': '1'}) 167 | func = FireflyFunction(dummy) 168 | d = func.get_multipart_formdata_inputs(req) 169 | assert d['data'].read().decode() == g.read() 170 | assert d['abc'] == 'hi' 171 | assert d['xyz'] == '1' 172 | 173 | def test_get_multipart_formdata_inputs_with_no_files(self): 174 | def dummy(): 175 | pass 176 | req = Request.blank('/filesize', POST={'abc': 'hi', 'xyz': 1}) 177 | func = FireflyFunction(dummy) 178 | d = func.get_multipart_formdata_inputs(req) 179 | assert d['abc'] == 'hi' 180 | assert d['xyz'] == '1' 181 | 182 | def test_get_content_type_present(self): 183 | req = Request.blank('/', headers={'Content-Type': 'multipart/form-data'}) 184 | func = FireflyFunction(dummy) 185 | content_type = func.get_content_type(req) 186 | assert content_type == 'multipart/form-data' 187 | 188 | def test_get_content_type_absent(self): 189 | req = Request.blank('/') 190 | func = FireflyFunction(dummy) 191 | content_type = func.get_content_type(req) 192 | assert content_type == 'application/octet-stream' 193 | 194 | @py2_only 195 | def test_generate_signature(self): 196 | def sample_function(x, one="hey", two=None, **kwargs): 197 | pass 198 | func = FireflyFunction(sample_function) 199 | assert len(func.sig) == 4 200 | assert func.sig[0]['name'] == 'x' 201 | assert func.sig[0]['kind'] == 'POSITIONAL_OR_KEYWORD' 202 | assert func.sig[1]['name'] == 'one' 203 | assert func.sig[1]['kind'] == 'POSITIONAL_OR_KEYWORD' 204 | assert func.sig[1]['default'] == 'hey' 205 | assert func.sig[2]['default'] == None 206 | assert func.sig[3]['name'] == 'kwargs' 207 | assert func.sig[3]['kind'] == 'VAR_KEYWORD' 208 | 209 | @py3_only 210 | def test_generate_signature_py3(self): 211 | # work-around to avoid syntax error in python 2 212 | code = 'def f(x, y=1, *, one="hey", two=None, **kwargs): pass' 213 | env = {} 214 | exec(code, env, env) 215 | f = env['f'] 216 | 217 | func = FireflyFunction(f) 218 | assert len(func.sig) == 5 219 | assert func.sig[0]['name'] == 'x' 220 | assert func.sig[0]['kind'] == 'POSITIONAL_OR_KEYWORD' 221 | assert func.sig[1]['default'] == 1 222 | assert func.sig[2]['name'] == 'one' 223 | assert func.sig[2]['kind'] == 'KEYWORD_ONLY' 224 | assert func.sig[2]['default'] == 'hey' 225 | assert func.sig[3]['default'] == None 226 | assert func.sig[4]['name'] == 'kwargs' 227 | assert func.sig[4]['kind'] == 'VAR_KEYWORD' 228 | -------------------------------------------------------------------------------- /tests/test_client.py: -------------------------------------------------------------------------------- 1 | import io 2 | import pytest 3 | import requests 4 | from firefly.client import Client, RemoteFunction, FireflyError 5 | from firefly.validator import ValidationError 6 | 7 | class MockResponse: 8 | def __init__(self, status_code, data, headers=None): 9 | self.status_code = status_code 10 | self.data = data 11 | self.headers = headers or {} 12 | self.headers.setdefault("Content-Type", "application/json") 13 | 14 | def json(self): 15 | return self.data 16 | 17 | def make_monkey_patch(status, return_data, mode='json'): 18 | def mock_post_response(url, json=None, data=None, files=None, headers=None, **kwargs): 19 | r = MockResponse(status_code=status, data=return_data, headers=headers) 20 | return r 21 | return mock_post_response 22 | 23 | class TestClass: 24 | def test_call_for_success_event(self, monkeypatch): 25 | monkeypatch.setattr(requests, "post", make_monkey_patch(200, 16)) 26 | monkeypatch.setattr(requests, "get", make_monkey_patch(200, {})) 27 | c = Client("http://127.0.0.1:8000") 28 | assert c.square(a=4) == 16 29 | 30 | def test_call_for_validation_error(self, monkeypatch): 31 | monkeypatch.setattr(requests, "post", make_monkey_patch(404, {"status": "not found"})) 32 | monkeypatch.setattr(requests, "get", make_monkey_patch(200, {})) 33 | c = Client("http://127.0.0.1:8000") 34 | with pytest.raises(FireflyError, message="Expected FireflyError"): 35 | c.sq(a=4) 36 | 37 | def test_call_for_validation_error(self, monkeypatch): 38 | monkeypatch.setattr(requests, "post", make_monkey_patch(422, {"error": "missing a required argument: 'a'"})) 39 | monkeypatch.setattr(requests, "get", make_monkey_patch(200, {})) 40 | c = Client("http://127.0.0.1:8000") 41 | with pytest.raises(ValidationError, message="Expected ValidationError"): 42 | c.square(b=4) 43 | 44 | def test_call_for_server_error(self, monkeypatch): 45 | monkeypatch.setattr(requests, "post", make_monkey_patch(500, {"error": "ValueError: Dummy Error"})) 46 | monkeypatch.setattr(requests, "get", make_monkey_patch(200, {})) 47 | c = Client("http://127.0.0.1:8000") 48 | with pytest.raises(FireflyError, message="Expected FireflyError"): 49 | c.square(a=4) 50 | 51 | def test_call_for_uncaught_exception(self, monkeypatch): 52 | monkeypatch.setattr(requests, "post", make_monkey_patch(502, "")) 53 | monkeypatch.setattr(requests, "get", make_monkey_patch(200, {})) 54 | c = Client("http://127.0.0.1:8000") 55 | with pytest.raises(FireflyError, message="Expected FireflyError"): 56 | c.square(a=4) 57 | 58 | def test_call_with_file_upload(self, monkeypatch): 59 | def filesize(data): 60 | return len(data.read()) 61 | f = io.StringIO(u"test file contents") 62 | monkeypatch.setattr(requests, "post", make_monkey_patch(200, "18")) 63 | monkeypatch.setattr(requests, "get", make_monkey_patch(200, {})) 64 | c = Client("http://127.0.0.1:8000") 65 | assert c.filesize(data=f) == "18" 66 | 67 | def test_decouple_files_with_files(self): 68 | f = io.StringIO(u"test file contents") 69 | c = Client("http://127.0.0.1:8000") 70 | kwargs = {'file': f} 71 | data, files = c.decouple_files(kwargs) 72 | assert data == {} 73 | assert files['file'] == f 74 | 75 | def test_decouple_files_with_no_files(self): 76 | c = Client("http://127.0.0.1:8000") 77 | kwargs = {'a': 1, 'b': 'c'} 78 | data, files = c.decouple_files(kwargs) 79 | assert data['a'] == 1 80 | assert data['b'] == 'c' 81 | assert files == {} 82 | 83 | def test_decouple_files_with_combined_input(self): 84 | f = io.StringIO(u"test file contents") 85 | c = Client("http://127.0.0.1:8000") 86 | kwargs = {'file': f, 'a': 1} 87 | data, files = c.decouple_files(kwargs) 88 | assert data['a'] == 1 89 | assert files['file'] == f 90 | 91 | def is_file_present(self): 92 | f = io.StringIO(u"test file contents") 93 | c = Client("http://127.0.0.1:8000") 94 | is_a_file = c.is_file(f) 95 | assert is_a_file == True 96 | 97 | def is_file_absent(self): 98 | c = Client("http://127.0.0.1:8000") 99 | is_a_file = c.is_file(1) 100 | assert is_a_file == False 101 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | import os 2 | from firefly.main import load_function 3 | 4 | def test_load_functions(): 5 | os.path.exists2 = os.path.exists 6 | path, name, func = load_function("os.path.exists2") 7 | assert path == "/exists2" 8 | assert name == "exists2" 9 | assert func == os.path.exists 10 | -------------------------------------------------------------------------------- /tests/test_validator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from firefly.validator import * 3 | 4 | def add(a, b): 5 | return a + b 6 | 7 | def test_equal_args(): 8 | try: 9 | validate_args(add, {"a": 1, "b":2}) 10 | except ValidationError: 11 | pytest.fail("Did not expect a ValidationError") 12 | 13 | def test_less_args(): 14 | with pytest.raises(ValidationError, message="Expected a ValidationError"): 15 | validate_args(add, {"a": 1}) 16 | 17 | def test_more_args(): 18 | with pytest.raises(ValidationError, message="Expected a ValidationError"): 19 | validate_args(add, {"a":1, "b":2, "c":3}) 20 | --------------------------------------------------------------------------------