├── .gitignore ├── .gitmodules ├── .travis.yml ├── CHANGELOG.md ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── benchmarks ├── aiohttp │ ├── micro.py │ └── requirements.txt ├── gevent │ ├── micro.py │ └── requirements.txt ├── golang-fasthttp │ ├── README.md │ └── micro.go ├── golang │ ├── README.md │ └── micro.go ├── japronro_c4.2xlarge.png ├── japronto │ └── micro.py ├── meinheld │ ├── micro.py │ └── requirements.txt ├── nodejs │ └── micro.js ├── results.ods ├── results.png ├── sanic │ ├── micro.py │ └── requirements.txt └── tornado │ ├── micro.py │ └── requirements.txt ├── build.py ├── cases ├── __init__.py ├── base.toml └── websites.toml ├── conftest.py ├── do_wrk.py ├── examples ├── 1_hello │ └── hello.py ├── 2_async │ └── async.py ├── 3_router │ └── router.py ├── 4_request │ └── request.py ├── 5_response │ └── response.py ├── 6_exceptions │ └── exceptions.py ├── 7_extend │ └── extend.py ├── 8_template │ ├── index.html │ └── template.py └── todo_api │ ├── .gitignore │ └── todo_api.py ├── integration_tests ├── __init__.py ├── common.py ├── drain.py ├── dump.py ├── experiments.py ├── generators.py ├── longrun.py ├── noleak.py ├── reaper.py ├── strategies.py ├── test_drain.py ├── test_noleak.py ├── test_perror.py ├── test_reaper.py └── test_request.py ├── misc ├── __init__.py ├── bootstrap.sh ├── buggers.py ├── cleanup_script.py ├── client.py ├── collector.py ├── cpu.py ├── do_perf.py ├── docker │ └── Dockerfile ├── parts.py ├── perf.md ├── pipeline.lua ├── report.py ├── requirements-test.txt ├── requirements.txt ├── rpm-requirements.txt ├── runpytest.py ├── simple.py ├── suppr.txt └── travis │ ├── before_install.sh │ ├── install.sh │ └── script.sh ├── setup.py ├── src ├── japronto │ ├── __init__.py │ ├── __main__.py │ ├── app │ │ └── __init__.py │ ├── capsule.c │ ├── capsule.h │ ├── common.h │ ├── cpu_features.c │ ├── cpu_features.h │ ├── parser │ │ ├── .gitignore │ │ ├── __init__.py │ │ ├── build_libpicohttpparser.py │ │ ├── cffiparser.py │ │ ├── cparser.c │ │ ├── cparser.gcda │ │ ├── cparser.h │ │ ├── cparser_ext.py │ │ └── test_parser.py │ ├── pipeline │ │ ├── __init__.py │ │ ├── cpipeline.c │ │ ├── cpipeline.h │ │ ├── cpipeline_ext.py │ │ └── test_pipeline.py │ ├── protocol │ │ ├── __init__.py │ │ ├── cprotocol.c │ │ ├── cprotocol.h │ │ ├── cprotocol_ext.py │ │ ├── creaper.c │ │ ├── creaper_ext.py │ │ ├── generator.c │ │ ├── generator.h │ │ ├── generator_ext.py │ │ ├── handler.py │ │ ├── null.py │ │ └── tracing.py │ ├── reloader.py │ ├── request │ │ ├── __init__.py │ │ ├── crequest.c │ │ ├── crequest.h │ │ └── crequest_ext.py │ ├── response │ │ ├── __init__.py │ │ ├── cresponse.c │ │ ├── cresponse.h │ │ ├── cresponse_ext.py │ │ ├── py.py │ │ └── reasons.h │ ├── router │ │ ├── __init__.py │ │ ├── analyzer.py │ │ ├── cmatcher.c │ │ ├── cmatcher.h │ │ ├── cmatcher_ext.py │ │ ├── match_dict.c │ │ ├── match_dict.h │ │ ├── matcher.py │ │ ├── route.py │ │ ├── test_analyzer.py │ │ ├── test_matcher.py │ │ └── test_route.py │ └── runner.py └── picohttpparser │ ├── build │ ├── picohttpparser.c │ └── picohttpparser.h └── tutorial ├── 1_hello.md ├── 2_async.md ├── 3_router.md ├── 4_request.md ├── 5_response.md ├── 6_exceptions.md ├── 7_extend.md └── 8_template.md /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .hypothesis 3 | .test 4 | coverage.info 5 | *.egg-info 6 | *.build.toml 7 | *.so 8 | *.o 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "misc/terryfy"] 2 | path = misc/terryfy 3 | url = https://github.com/MacPython/terryfy/ 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | sudo: required 4 | 5 | services: 6 | - docker 7 | 8 | env: 9 | global: 10 | - secure: "HO3zCuv0FtNFTQ7kkBpIqKYAZW8sYZPfc1ROk6+ChoxufXcu529CTKNAr3KklfZCbMHiZKc3W83N7x9B/L2rtSuBQvJPPgVtIlaVKRyWWnY4nqrpwKEoOLUd3RjpAMfCB09sXQ2aTfQV8Ds5Zk+cF7R2toI6s2s4vymXvCLvfugrtO4sd91frSDv/fzjEEKOIeey8KXtPAPPFv6v64OScksPt1oCsVOPDtkZ7q0KSIzS9JN6BvM9oafPt9MaFPH84ITtdPMTjgQOQ+YFe8YBwgjkV/cX9rNs+vzSP6Bm2NQ9/xxd8XTDj6ukuEYD5HQi26IS6ddRyVGsn6/WRZx6/kQboJKh5r5pa4OAHPWnPirRWPQW46HI2iknAGTFWh8ARX2R208mK1vbQ66J+9zQDnkjMXGgX67gWyWQWfxwVHFsPyQEiSHHh2vEBkhs1+tqtvp7Ktnc+uCxXn0v/Humu3OvFSBxSXfyjvE9uUOGyB2zDwqmxLQQ5ftKAcGfLOaSqauJ1vQy1CWc5bROCn8aoch5iRf/tcX85TUDirAgAp3OUdt3VwcRNY+Fci7IU50gn2rghJWFzB5Zz9p1ShnZxIaD5GEPE45ju4UIpwYbs8iSqh+/RS8sR2Ffzx4M+6QJjj1BJABdtVPS9Jn5OkbuSdBW0K+MuLtmtbg4WLXv6+E=" 11 | 12 | jobs: 13 | include: 14 | - python: 3.5 15 | env: VERSION=3.5.3 16 | - python: 3.6 17 | env: VERSION=3.6.0 18 | - python: 3.7 19 | env: VERSION=3.7.1 20 | - python: 3.8 21 | env: VERSION=3.8.0 22 | 23 | before_install: source misc/travis/before_install.sh 24 | 25 | install: source misc/travis/install.sh 26 | 27 | script: misc/travis/script.sh 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 0.1.1 - Feb 9 2017 2 | ------------------ 3 | 4 | - Native support for OSX 5 | - Support for older hardware without SSE4.2 6 | - Better crash info with faulthandler 7 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Paweł Piotr Przeradowski 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE.txt src/picohttpparser/picohttpparser.c src/picohttpparser/picohttpparser.h 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Japronto! 2 | 3 | [![irc: #japronto](https://img.shields.io/badge/irc-%23japronto-brightgreen.svg)](https://webchat.freenode.net/?channels=japronto) 4 | [![Gitter japronto/Lobby](https://badges.gitter.im/japronto/Lobby.svg)](https://gitter.im/japronto/Lobby) [![Build Status](https://travis-ci.org/squeaky-pl/japronto.svg?branch=master)](https://travis-ci.org/squeaky-pl/japronto) [![PyPI](https://img.shields.io/pypi/v/japronto.svg)](https://pypi.python.org/pypi/japronto) [![PyPI version](https://img.shields.io/pypi/pyversions/japronto.svg)](https://pypi.python.org/pypi/japronto/) 5 | 6 | __There is no new project development happening at the moment, but it's not abandoned either. Pull requests and new maintainers are welcome__. 7 | 8 | __If you are a novice Python programmer, you don't like plumbing yourself or you don't have basic understanding of C, this project is not probably what you are looking for__. 9 | 10 | Japronto (from Portuguese "já pronto" /ˈʒa pɾõtu/ meaning "already done") is a __screaming-fast__, __scalable__, __asynchronous__ 11 | Python 3.5+ HTTP __toolkit__ integrated with __pipelining HTTP server__ based on [uvloop](https://github.com/MagicStack/uvloop) and [picohttpparser](https://github.com/h2o/picohttpparser). It's targeted at speed enthusiasts, people who like 12 | plumbing and early adopters. 13 | 14 | You can read more in the [release announcement on medium](https://medium.com/@squeaky_pl/million-requests-per-second-with-python-95c137af319) 15 | 16 | Performance 17 | ----------- 18 | 19 | Here's a chart to help you imagine what kind of things you can do with Japronto: 20 | 21 | ![Requests per second](benchmarks/results.png) 22 | 23 | As user @heppu points out Go’s stdlib HTTP server can be 12% faster than the graph shows when written more carefully. Also there is the awesome fasthttp server for Go that apparently is only 18% slower than Japronto in this particular benchmark. Awesome! For details see https://github.com/squeaky-pl/japronto/pull/12 and https://github.com/squeaky-pl/japronto/pull/14. 24 | 25 | These results of a simple "Hello world" application were obtained on AWS c4.2xlarge instance. To be fair all the contestants (including Go) were running single worker process. Servers were load tested using [wrk](https://github.com/wg/wrk) with 1 thread, 100 connections and 24 simultaneous (pipelined) requests per connection (cumulative parallelism of 2400 requests). 26 | 27 | The source code for the benchmark can be found in [benchmarks](benchmarks) directory. 28 | 29 | The server is written in hand tweaked C trying to take advantage of modern CPUs. It relies on picohttpparser for header & 30 | chunked-encoding parsing while uvloop provides asynchronous I/O. It also tries to save up on 31 | system calls by combining writes together when possible. 32 | 33 | Early preview 34 | ------------- 35 | 36 | This is an early preview with alpha quality implementation. APIs are provisional meaning that they will change between versions and more testing is needed. Don't use it for anything serious for now and definitely don't use it in production. Please try it though and report back feedback. If you are shopping for your next project's framework I would recommend [Sanic](https://github.com/channelcat/sanic). 37 | 38 | At the moment the work is focused on CPython but I have PyPy on my radar, though I am not gonna look into it until PyPy reaches 3.5 compatibility somewhere later this year and most known JIT regressions are removed. 39 | 40 | Hello world 41 | ----------- 42 | 43 | Here is how a simple web application looks like in Japronto: 44 | 45 | ```python 46 | from japronto import Application 47 | 48 | 49 | def hello(request): 50 | return request.Response(text='Hello world!') 51 | 52 | 53 | app = Application() 54 | app.router.add_route('/', hello) 55 | app.run(debug=True) 56 | ``` 57 | 58 | Tutorial 59 | -------- 60 | 61 | 1. [Getting started](tutorial/1_hello.md) 62 | 2. [Asynchronous handlers](tutorial/2_async.md) 63 | 3. [Router](tutorial/3_router.md) 64 | 4. [Request object](tutorial/4_request.md) 65 | 5. [Response object](tutorial/5_response.md) 66 | 6. [Handling exceptions](tutorial/6_exceptions.md) 67 | 7. [Extending request](tutorial/7_extend.md) 68 | 69 | Features 70 | -------- 71 | 72 | - HTTP 1.x implementation with support for chunked uploads 73 | - Full support for HTTP pipelining 74 | - Keep-alive connections with configurable reaper 75 | - Support for synchronous and asynchronous views 76 | - Master-multiworker model based on forking 77 | - Support for code reloading on changes 78 | - Simple routing 79 | 80 | License 81 | ------- 82 | 83 | This software is distributed under [MIT License](https://en.wikipedia.org/wiki/MIT_License). This is a very permissive license that lets you use this software for any 84 | commercial and non-commercial work. Full text of the license is 85 | included in [LICENSE.txt](LICENSE.txt) file. 86 | 87 | The source distribution of this software includes a copy of picohttpparser which is distributed under MIT license as well. 88 | -------------------------------------------------------------------------------- /benchmarks/aiohttp/micro.py: -------------------------------------------------------------------------------- 1 | from aiohttp import web 2 | import asyncio 3 | import uvloop 4 | 5 | 6 | loop = uvloop.new_event_loop() 7 | asyncio.set_event_loop(loop) 8 | 9 | 10 | async def hello(request): 11 | return web.Response(text='Hello world!') 12 | 13 | app = web.Application(loop=loop) 14 | app.router.add_route('GET', '/', hello) 15 | 16 | web.run_app(app, port=8080, access_log=None) 17 | -------------------------------------------------------------------------------- /benchmarks/aiohttp/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==1.2.0 2 | -------------------------------------------------------------------------------- /benchmarks/gevent/micro.py: -------------------------------------------------------------------------------- 1 | from gevent.pywsgi import WSGIServer 2 | 3 | 4 | def hello(environ, start_response): 5 | if(environ['PATH_INFO'] == '/' and environ['REQUEST_METHOD'] == 'GET'): 6 | status = '200 OK' 7 | text = "Hello world!" 8 | else: 9 | status = '404 Not Found' 10 | text = "Not Found" 11 | 12 | body = text.encode('utf-8') 13 | response_headers = [ 14 | ('Content-type', 'text/plain; charset=utf-8'), 15 | ('Content-Length', str(len(body)))] 16 | start_response(status, response_headers) 17 | return [body] 18 | 19 | 20 | WSGIServer(('0.0.0.0', 8080), hello, log=None).serve_forever() 21 | -------------------------------------------------------------------------------- /benchmarks/gevent/requirements.txt: -------------------------------------------------------------------------------- 1 | gevent==1.2.1 2 | -------------------------------------------------------------------------------- /benchmarks/golang-fasthttp/README.md: -------------------------------------------------------------------------------- 1 | ``` 2 | go build . 3 | GOMAXPROCS=1 ./bin 4 | ``` 5 | -------------------------------------------------------------------------------- /benchmarks/golang-fasthttp/micro.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/valyala/fasthttp" 4 | 5 | func hello(ctx *fasthttp.RequestCtx) { 6 | if string(ctx.Path()) != "/" { 7 | ctx.SetStatusCode(404) 8 | ctx.WriteString("Not Found") 9 | return 10 | } 11 | ctx.WriteString("Hello world!") 12 | } 13 | 14 | func main() { 15 | fasthttp.ListenAndServe("0.0.0.0:8080", hello) 16 | } 17 | -------------------------------------------------------------------------------- /benchmarks/golang/README.md: -------------------------------------------------------------------------------- 1 | ``` 2 | go build . 3 | GOMAXPROCS=1 ./bin 4 | ``` 5 | -------------------------------------------------------------------------------- /benchmarks/golang/micro.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "net/http" 4 | 5 | var ( 6 | helloResp = []byte("Hello world!") 7 | notFoundResp = []byte("Not Found") 8 | ) 9 | 10 | func hello(w http.ResponseWriter, r *http.Request) { 11 | if r.URL.Path != "/" { 12 | w.WriteHeader(http.StatusNotFound) 13 | w.Write(notFoundResp) 14 | return 15 | } 16 | w.Write(helloResp) 17 | } 18 | 19 | func main() { 20 | http.HandleFunc("/", hello) 21 | http.ListenAndServe("0.0.0.0:8080", nil) 22 | } 23 | -------------------------------------------------------------------------------- /benchmarks/japronro_c4.2xlarge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/benchmarks/japronro_c4.2xlarge.png -------------------------------------------------------------------------------- /benchmarks/japronto/micro.py: -------------------------------------------------------------------------------- 1 | from japronto import Application 2 | 3 | 4 | def hello(request): 5 | return request.Response(text='Hello world!') 6 | 7 | 8 | app = Application() 9 | 10 | r = app.router 11 | r.add_route('/', hello, method='GET') 12 | 13 | app.run() 14 | -------------------------------------------------------------------------------- /benchmarks/meinheld/micro.py: -------------------------------------------------------------------------------- 1 | from meinheld import server 2 | 3 | 4 | def hello(environ, start_response): 5 | if(environ['PATH_INFO'] == '/' and environ['REQUEST_METHOD'] == 'GET'): 6 | status = '200 OK' 7 | text = "Hello world!" 8 | else: 9 | status = '404 Not Found' 10 | text = "Not Found" 11 | 12 | body = text.encode('utf-8') 13 | response_headers = [ 14 | ('Content-type', 'text/plain; charset=utf-8'), 15 | ('Content-Length', str(len(body)))] 16 | start_response(status, response_headers) 17 | return [body] 18 | 19 | 20 | server.listen(('0.0.0.0', 8080)) 21 | server.set_access_logger(None) 22 | server.set_keepalive(1) 23 | server.run(hello) 24 | -------------------------------------------------------------------------------- /benchmarks/meinheld/requirements.txt: -------------------------------------------------------------------------------- 1 | meinheld==0.6.1 2 | -------------------------------------------------------------------------------- /benchmarks/nodejs/micro.js: -------------------------------------------------------------------------------- 1 | const http = require('http'); 2 | 3 | 4 | var srv = http.createServer( (req, res) => { 5 | res.sendDate = false; 6 | if(req.url == '/') { 7 | data = 'Hello world!' 8 | status = 200 9 | } else { 10 | data = 'Not Found' 11 | status = 404 12 | } 13 | res.writeHead(status, { 14 | 'Content-Type': 'text/plain; encoding=utf-8', 15 | 'Content-Length': data.length}); 16 | res.end(data); 17 | }); 18 | 19 | 20 | srv.listen(8080, '0.0.0.0'); 21 | -------------------------------------------------------------------------------- /benchmarks/results.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/benchmarks/results.ods -------------------------------------------------------------------------------- /benchmarks/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/benchmarks/results.png -------------------------------------------------------------------------------- /benchmarks/sanic/micro.py: -------------------------------------------------------------------------------- 1 | from sanic import Sanic 2 | from sanic.response import text 3 | 4 | app = Sanic(__name__) 5 | 6 | 7 | @app.route("/") 8 | async def hello(request): 9 | return text("Hello world!") 10 | 11 | app.run(host="0.0.0.0", port=8080) 12 | -------------------------------------------------------------------------------- /benchmarks/sanic/requirements.txt: -------------------------------------------------------------------------------- 1 | sanic==0.2.0 2 | -------------------------------------------------------------------------------- /benchmarks/tornado/micro.py: -------------------------------------------------------------------------------- 1 | from tornado import web 2 | from tornado.httputil import HTTPHeaders, responses 3 | from tornado.platform.asyncio import AsyncIOMainLoop 4 | import asyncio 5 | import uvloop 6 | 7 | 8 | loop = uvloop.new_event_loop() 9 | asyncio.set_event_loop(loop) 10 | AsyncIOMainLoop().install() 11 | 12 | 13 | class MainHandler(web.RequestHandler): 14 | def get(self): 15 | self.write('Hello world!') 16 | 17 | # skip calculating ETag, ~8% faster 18 | def set_etag_header(self): 19 | pass 20 | 21 | def check_etag_header(self): 22 | return False 23 | 24 | # torando sends Server and Date headers by default, ~4% faster 25 | def clear(self): 26 | self._headers = HTTPHeaders( 27 | {'Content-Type': 'text/plain; charset=utf-8'}) 28 | self._write_buffer = [] 29 | self._status_code = 200 30 | self._reason = responses[200] 31 | 32 | 33 | app = web.Application([('/', MainHandler)]) 34 | 35 | app.listen(8080) 36 | 37 | loop.run_forever() 38 | -------------------------------------------------------------------------------- /benchmarks/tornado/requirements.txt: -------------------------------------------------------------------------------- 1 | tornado==4.4.2 2 | -------------------------------------------------------------------------------- /cases/__init__.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | import glob 3 | import os.path 4 | 5 | import pytoml 6 | import pytest 7 | 8 | 9 | testcase_fields = 'data,method,path,version,headers,body,error,disconnect' 10 | 11 | HttpTestCase = namedtuple('HTTPTestCase', testcase_fields) 12 | 13 | 14 | def parse_casesel(suite, casesel): 15 | for casespec in casesel.split('+'): 16 | *transforms, case = casespec.split(':') 17 | if case.endswith('!'): 18 | transforms.append('!') 19 | case = case[:-1] 20 | case = suite[case] 21 | 22 | for transform in reversed(transforms): 23 | func, *args = transform.split() 24 | case = transorm_dict[func](case, *args) 25 | 26 | yield case 27 | 28 | 29 | def parametrize_cases(suite, *args): 30 | suite = suites[suite] 31 | cases_list = [ 32 | list(parse_casesel(suite, sel)) for sel in args] 33 | return pytest.mark.parametrize('cases', cases_list, ids=args) 34 | 35 | 36 | def load_casefile(path): 37 | result = {} 38 | 39 | with open(path) as casefile: 40 | cases = pytoml.load(casefile) 41 | 42 | for case_name, case_data in cases.items(): 43 | case_data['data'] = case_data['data'].encode('utf-8') 44 | case_data['body'] = case_data['body'].encode('utf-8') \ 45 | if 'body' in case_data else None 46 | case_data['disconnect'] = False 47 | case = HttpTestCase._make( 48 | case_data.get(f) for f in testcase_fields.split(',')) 49 | result[case_name] = case 50 | 51 | return result 52 | 53 | 54 | def load_cases(): 55 | cases = {} 56 | 57 | for filename in glob.glob('cases/*.toml'): 58 | suite_name, _ = os.path.splitext(os.path.basename(filename)) 59 | cases[suite_name] = load_casefile(filename) 60 | 61 | return cases 62 | 63 | 64 | def keep_alive(case): 65 | headers = case.headers.copy() 66 | headers['Connection'] = 'keep-alive' 67 | # if case.body is not None \ 68 | # and headers.get('Transfer-Encoding', 'identity') == 'identity': 69 | # headers['Content-Length'] = str(len(case.body)) 70 | 71 | return update_case(case, headers) 72 | 73 | def close(case): 74 | headers = case.headers.copy() 75 | headers['Connection'] = 'close' 76 | 77 | return update_case(case, headers) 78 | 79 | 80 | def should_keep_alive(case): 81 | return case.headers.get( 82 | 'Connection', 83 | 'close' if case.version == '1.0' else 'keep-alive') == 'keep-alive' 84 | 85 | 86 | def set_error(case, error): 87 | return update_case(case, error=error) 88 | 89 | 90 | def disconnect(case): 91 | return update_case(case, disconnect=True) 92 | 93 | 94 | def update_case(case, headers=False, error=False, disconnect=None): 95 | data = False 96 | if headers: 97 | data = bytearray() 98 | status = case.method + ' ' + case.path + ' HTTP/' + case.version + '\r\n' 99 | data += status.encode('ascii') 100 | for name, value in headers.items(): 101 | data += name.encode('ascii') + b': ' + value.encode('latin1') + b'\r\n' 102 | data += b'\r\n' 103 | if case.body: 104 | data += case.body 105 | 106 | headers = headers or case.headers 107 | data = data or case.data 108 | error = error or case.error 109 | disconnect = disconnect if disconnect is not None else case.disconnect 110 | 111 | return case._replace( 112 | headers=headers, error=error, disconnect=disconnect, 113 | data=bytes(data)) 114 | 115 | 116 | transorm_dict = { 117 | 'keep': keep_alive, 118 | 'close': close, 119 | 'e': set_error, 120 | '!': disconnect 121 | } 122 | 123 | 124 | suites = load_cases() 125 | globals().update(suites) 126 | -------------------------------------------------------------------------------- /cases/base.toml: -------------------------------------------------------------------------------- 1 | [10msg] 2 | data = """\ 3 | POST \ 4 | /wp-content/uploads/2010/03/hello-kitty-darth-vader-pink.jpg \ 5 | HTTP/1.0\r\n\ 6 | HOST: www.kittyhell.com\r\n\ 7 | User-Agent: Mozilla/5.0 \ 8 | (Macintosh; U; Intel Mac OS X 10.6; ja-JP-mac; rv:1.9.2.3) \ 9 | Gecko/20100401 Firefox/3.6.3 Pathtraq/0.9\r\n\ 10 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n\ 11 | Accept-Language: ja,en-us;q=0.7,en;q=0.3\r\n\ 12 | Accept-Encoding: gzip,deflate\r\n\ 13 | Accept-Charset: Shift_JIS,utf-8;q=0.7,*;q=0.7\r\n\ 14 | Keep-Alive: 115\r\n\ 15 | Cookie: wp_ozh_wsa_visits=2; \ 16 | wp_ozh_wsa_visit_lasttime=xxxxxxxxxx; \ 17 | __utma=xxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.x; \ 18 | __utmz=xxxxxxxxx.xxxxxxxxxx.x.x.utmccn=(referral)|utmcsr=reader.livedoor.com|utmcct=/reader/|utmcmd=referral\r\n\ 19 | Content-Length: 11\r\n\ 20 | \r\n\ 21 | Hello there\ 22 | """ 23 | 24 | method = "POST" 25 | path = "/wp-content/uploads/2010/03/hello-kitty-darth-vader-pink.jpg" 26 | version = "1.0" 27 | body = "Hello there" 28 | [10msg.headers] 29 | Host = "www.kittyhell.com" 30 | User-Agent = """Mozilla/5.0 \ 31 | (Macintosh; U; Intel Mac OS X 10.6; ja-JP-mac; rv:1.9.2.3) \ 32 | Gecko/20100401 Firefox/3.6.3 Pathtraq/0.9""" 33 | Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" 34 | Accept-Language = "ja,en-us;q=0.7,en;q=0.3" 35 | Accept-Encoding = "gzip,deflate" 36 | Accept-Charset = "Shift_JIS,utf-8;q=0.7,*;q=0.7" 37 | Keep-Alive = "115" 38 | Cookie = """wp_ozh_wsa_visits=2; \ 39 | wp_ozh_wsa_visit_lasttime=xxxxxxxxxx; \ 40 | __utma=xxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.x; \ 41 | __utmz=xxxxxxxxx.xxxxxxxxxx.x.x.utmccn=(referral)|utmcsr=reader.livedoor.com|utmcct=/reader/|utmcmd=referral""" 42 | Content-Length = "11" 43 | 44 | [10get] 45 | data = "GET / HTTP/1.0\r\nHost: www.example.com\r\n\r\n" 46 | method = "GET" 47 | path = "/" 48 | version = "1.0" 49 | headers = { Host = "www.example.com" } 50 | 51 | 52 | [10malformed_headers1] 53 | data = "GET / HTTP 1.0" 54 | error = "malformed_headers" 55 | 56 | 57 | [10malformed_headers2] 58 | data = "GET / HTTP/2" 59 | error = "malformed_headers" 60 | 61 | 62 | [10incomplete_headers] 63 | data = "GET / HTTP/1.0\r\nH" 64 | error = "incomplete_headers" 65 | 66 | 67 | [11get] 68 | data = "GET /index.html HTTP/1.1\r\n\r\n" 69 | method = "GET" 70 | path = "/index.html" 71 | version = "1.1" 72 | headers = {} 73 | # body = null 74 | 75 | 76 | [11getmsg] 77 | data = """\ 78 | GET /body HTTP/1.1\r\n\ 79 | Content-Length: 12\r\n\ 80 | \r\n\ 81 | Hello World!\ 82 | """ 83 | method = "GET" 84 | path = "/body" 85 | version = "1.1" 86 | headers = { Content-Length = "12" } 87 | body = "Hello World!" 88 | 89 | 90 | [11msg] 91 | data = "POST /login HTTP/1.1\r\nContent-Length: 5\r\n\r\nHello" 92 | method = "POST" 93 | path = "/login" 94 | version = "1.1" 95 | headers = { Content-Length = "5" } 96 | body = "Hello" 97 | 98 | 99 | [11msgzero] 100 | data = "POST /zero HTTP/1.1\r\nContent-Length: 0\r\n\r\n" 101 | method = "POST" 102 | path = "/zero" 103 | version = "1.1" 104 | headers = { Content-Length = "0" } 105 | body = "" 106 | 107 | 108 | [11clincomplete_headers] 109 | data = """\ 110 | POST / HTTP/1.1\r\n\ 111 | Content-Length: 3\r\n\ 112 | I""" 113 | error = "incomplete_headers" 114 | 115 | 116 | [11clincomplete_body] 117 | data = "POST / HTTP/1.1\r\nContent-Length: 5\r\n\r\nI" 118 | method = "POST" 119 | path = "/" 120 | version = "1.1" 121 | headers = { Content-Length = "5" } 122 | error = "incomplete_body" 123 | 124 | [11clinvalid1] 125 | data = "POST / HTTP/1.1\r\nContent-Length: asd\r\n\r\n" 126 | method = "POST" 127 | path = "/" 128 | version = "1.1" 129 | headers = { Content-Length = "asd" } 130 | error = "invalid_headers" 131 | 132 | [11clinvalid2] 133 | data = "POST / HTTP/1.1\r\nContent-Length: +5\r\n\r\n" 134 | method = "POST" 135 | path = "/" 136 | version = "1.1" 137 | headers = { Content-Length = "+5" } 138 | error = "invalid_headers" 139 | 140 | [11clinvalid3] 141 | data = "POST / HTTP/1.1\r\nContent-Length: -5\r\n\r\n" 142 | method = "POST" 143 | path = "/" 144 | version = "1.1" 145 | headers = { Content-Length = "+5" } 146 | error = "invalid_headers" 147 | 148 | [11clinvalid4] 149 | data = "POST / HTTP/1.1\r\nContent-Length: 4f\r\n\r\n" 150 | method = "POST" 151 | path = "/" 152 | version = "1.1" 153 | headers = { Content-Length = "4f" } 154 | error = "invalid_headers" 155 | 156 | [11clinvalid5] 157 | data = "POST / HTTP/1.1\r\nContent-Length: \r\n\r\n" 158 | method = "POST" 159 | path = "/" 160 | version = "1.1" 161 | headers = { Content-Length = "" } 162 | error = "invalid_headers" 163 | 164 | 165 | [11chunked1] 166 | data = """\ 167 | POST /chunked HTTP/1.1\r\n\ 168 | Transfer-Encoding: chunked\r\n\ 169 | \r\n\ 170 | 4\r\n\ 171 | Wiki\r\n\ 172 | 5\r\n\ 173 | pedia\r\n\ 174 | E\r\n in\r\n\ 175 | \r\n\ 176 | chunks.\r\n\ 177 | 0\r\n\ 178 | \r\n\ 179 | """ 180 | method = "POST" 181 | path = "/chunked" 182 | version = "1.1" 183 | headers = { Transfer-Encoding = "chunked" } 184 | body = "Wikipedia in\r\n\r\nchunks." 185 | 186 | [11chunkedzero] 187 | data = """ 188 | PUT /zero HTTP/1.1\r\n\ 189 | Transfer-Encoding: chunked\r\n\ 190 | \r\n\ 191 | 0\r\n\ 192 | \r\n\ 193 | """ 194 | method = "PUT" 195 | path = "/zero" 196 | version = "1.1" 197 | headers = { Transfer-Encoding = "chunked" } 198 | body = "" 199 | 200 | [11chunked2] 201 | data = """\ 202 | POST /chunked HTTP/1.1\r\n\ 203 | Transfer-Encoding: chunked\r\n\ 204 | \r\n\ 205 | 1;token=123;x=3\r\n\ 206 | r\r\n\ 207 | 0\r\n\ 208 | \r\n\ 209 | """ 210 | method = "POST" 211 | path = "/chunked" 212 | version = "1.1" 213 | headers = { Transfer-Encoding = "chunked" } 214 | body = "r" 215 | 216 | 217 | [11chunked3] 218 | data = """\ 219 | POST / HTTP/1.1\r\n\ 220 | Transfer-Encoding: chunked\r\n\ 221 | \r\n\ 222 | 000002\r\n\ 223 | ab\r\n\ 224 | 0;q=1\r\n\ 225 | This: is trailer header\r\n\ 226 | \r\n\ 227 | """ 228 | method = "POST" 229 | path = "/" 230 | version = "1.1" 231 | headers = { Transfer-Encoding = "chunked" } 232 | body = "ab" 233 | 234 | 235 | [11chunkedincomplete_body] 236 | data = "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n10\r\nasd" 237 | method = "POST" 238 | path = "/" 239 | version = "1.1" 240 | headers = { Transfer-Encoding = "chunked" } 241 | error = "incomplete_body" 242 | 243 | 244 | [11chunkedmalformed_body] 245 | data = "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n1x\r\nhello" 246 | method = "POST" 247 | path = "/" 248 | version = "1.1" 249 | headers = { Transfer-Encoding = "chunked" } 250 | error = "malformed_body" 251 | -------------------------------------------------------------------------------- /cases/websites.toml: -------------------------------------------------------------------------------- 1 | [github] 2 | data = """\ 3 | GET / HTTP/1.1\r\n\ 4 | Host: github.com\r\n\ 5 | User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0\r\n\ 6 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n\ 7 | Accept-Language: en-US,en;q=0.5\r\n\ 8 | Accept-Encoding: gzip, deflate, br\r\n\ 9 | Cookie: _octo=GHx; logged_in=yes; _ga=GAx; user_session=x; dotcom_user=x; __Host-user_session_same_site=x; _gh_sess=x; tz=America%2FSao_Paulo; _gat=x\r\n\ 10 | Connection: keep-alive\r\n\ 11 | Upgrade-Insecure-Requests: 1\r\n\ 12 | Pragma: no-cache\r\n\ 13 | Cache-Control: no-cache\r\n\ 14 | \r\n\ 15 | """ 16 | 17 | [google] 18 | data = """\ 19 | GET / HTTP/1.1\r\n\ 20 | Host: google.com\r\n\ 21 | Connection: keep-alive\r\n\ 22 | Upgrade-Insecure-Requests: 1\r\n\ 23 | User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\r\n\ 24 | X-Client-Data: Qwerty\r\n\ 25 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n\ 26 | Accept-Encoding: gzip, deflate, sdch\r\n\ 27 | Accept-Language: en-US,en;q=0.8,es;q=0.6\r\n\ 28 | Cookie: NID=x; SID=x; HSID=x; APISID=x\r\n\ 29 | \r\n\ 30 | """ 31 | 32 | [amazon] 33 | data = """\ 34 | GET / HTTP/1.1\r\n\ 35 | Host: www.amazon.com\r\n\ 36 | Connection: keep-alive\r\n\ 37 | Upgrade-Insecure-Requests: 1\r\n\ 38 | User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.50 Safari/537.36 OPR/41.0.2353.23 (Edition beta)\r\n\ 39 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n\ 40 | Accept-Encoding: gzip, deflate, lzma, sdch, br\r\n\ 41 | Accept-Language: en-US,en;q=0.8\r\n\ 42 | \r\n\ 43 | """ 44 | 45 | [xkcd] 46 | data = """\ 47 | GET /comics/mushrooms.png HTTP/1.1\r\n\ 48 | Host: imgs.xkcd.com\r\n\ 49 | Connection: keep-alive\r\n\ 50 | Upgrade-Insecure-Requests: 1\r\n\ 51 | User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.50 Safari/537.36 OPR/41.0.2353.23 (Edition beta)\r\n\ 52 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n\ 53 | Accept-Encoding: gzip, deflate, lzma, sdch\r\n\ 54 | Accept-Language: en-US,en;q=0.8\r\n\ 55 | \r\n\ 56 | """ 57 | 58 | [4chan] 59 | data = """\ 60 | GET /image/favicon.ico HTTP/1.1\r\n\ 61 | Host: s.4cdn.org\r\n\ 62 | Connection: keep-alive\r\n\ 63 | User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\r\n\ 64 | Accept: */*\r\n\ 65 | Referer: http://www.4chan.org/\r\n\ 66 | Accept-Encoding: gzip, deflate, sdch\r\n\ 67 | Accept-Language: en-US,en;q=0.8,es;q=0.6\r\n\ 68 | \r\n\ 69 | """ 70 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | import os 4 | import shutil 5 | 6 | 7 | builds = [] 8 | coverages = set() 9 | 10 | 11 | def add_build(mark): 12 | global builds 13 | args, kwargs = list(mark.args), mark.kwargs.copy() 14 | kwargs.pop('coverage', None) 15 | cfg = args, kwargs 16 | if cfg not in builds: 17 | builds.append(cfg) 18 | 19 | 20 | def execute_builds(): 21 | common_options = ['--coverage', '-d', '--sanitize'] 22 | for args, kwargs in builds: 23 | build_options = args[:] 24 | build_options.extend(['--dest', kwargs.get('dest', '.test')]) 25 | if 'kit' not in kwargs: 26 | build_options.extend(['--kit', 'platform']) 27 | build_options.extend(common_options) 28 | 29 | print('Executing build', *build_options) 30 | subprocess.check_call([sys.executable, 'build.py', *build_options]) 31 | 32 | 33 | def add_coverage(mark): 34 | dest = mark.kwargs.get('dest', '.test') 35 | coverages.add(dest) 36 | 37 | 38 | def setup_coverage(): 39 | if coverages: 40 | print('Setting up C coverage for', *coverages) 41 | 42 | for dest in coverages: 43 | subprocess.check_call([ 44 | 'lcov', '--base-directory', '.', '--directory', 45 | dest + '/.build/temp', '--zerocounters', '-q']) 46 | 47 | 48 | def make_coverage(): 49 | for dest in coverages: 50 | try: 51 | os.unlink(dest + '/coverage.info') 52 | except FileNotFoundError: 53 | pass 54 | 55 | subprocess.check_call([ 56 | 'lcov', '--base-directory', '.', '--directory', 57 | dest + '/.build/temp', '-c', '-o', dest + '/coverage.info', '-q']) 58 | subprocess.check_call([ 59 | 'lcov', '--remove', dest + '/coverage.info', 60 | '/usr*', '-o', 'coverage.info', '-q']) 61 | 62 | try: 63 | shutil.rmtree(dest + '/coverage_report') 64 | except FileNotFoundError: 65 | pass 66 | 67 | subprocess.check_call([ 68 | 'genhtml', '-o', dest + '/coverage_report', 69 | dest + '/coverage.info', '-q' 70 | ]) 71 | 72 | print('C coverage report saved in', 73 | dest + '/coverage_report/index.html') 74 | 75 | 76 | def pytest_itemcollected(item): 77 | needs_build = item.get_closest_marker('needs_build') 78 | if needs_build: 79 | add_build(needs_build) 80 | if needs_build and needs_build.kwargs.get('coverage'): 81 | add_coverage(needs_build) 82 | 83 | 84 | def pytest_collection_modifyitems(config, items): 85 | execute_builds() 86 | setup_coverage() 87 | 88 | 89 | def pytest_unconfigure(): 90 | make_coverage() 91 | -------------------------------------------------------------------------------- /do_wrk.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | import asyncio as aio 4 | import os 5 | from asyncio.subprocess import PIPE, STDOUT 6 | import statistics 7 | 8 | import uvloop 9 | import psutil 10 | 11 | from misc import cpu 12 | from misc import buggers 13 | 14 | 15 | def run_wrk(loop, endpoint=None): 16 | endpoint = endpoint or 'http://localhost:8080' 17 | wrk_fut = aio.create_subprocess_exec( 18 | './wrk', '-t', '1', '-c', '100', '-d', '2', '-s', 'misc/pipeline.lua', 19 | endpoint, stdout=PIPE, stderr=STDOUT) 20 | 21 | wrk = loop.run_until_complete(wrk_fut) 22 | 23 | lines = [] 24 | while 1: 25 | line = loop.run_until_complete(wrk.stdout.readline()) 26 | if line: 27 | line = line.decode('utf-8') 28 | lines.append(line) 29 | if line.startswith('Requests/sec:'): 30 | rps = float(line.split()[-1]) 31 | else: 32 | break 33 | 34 | retcode = loop.run_until_complete(wrk.wait()) 35 | if retcode != 0: 36 | print('\r\n'.join(lines)) 37 | 38 | return rps 39 | 40 | 41 | def cpu_usage(p): 42 | return p.cpu_percent() + sum(c.cpu_percent() for c in p.children()) 43 | 44 | 45 | def connections(process): 46 | return len( 47 | set(c.fd for c in process.connections()) | 48 | set(c.fd for p in process.children() for c in p.connections())) 49 | 50 | 51 | def memory(p): 52 | return p.memory_percent('uss') \ 53 | + sum(c.memory_percent('uss') for c in p.children()) 54 | 55 | 56 | if __name__ == '__main__': 57 | buggers.silence() 58 | loop = uvloop.new_event_loop() 59 | 60 | argparser = argparse.ArgumentParser('do_wrk') 61 | argparser.add_argument('-s', dest='server', default='') 62 | argparser.add_argument('-e', dest='endpoint') 63 | argparser.add_argument('--pid', dest='pid', type=int) 64 | argparser.add_argument( 65 | '--no-cpu', dest='cpu_change', default=True, 66 | action='store_const', const=False) 67 | 68 | args = argparser.parse_args(sys.argv[1:]) 69 | 70 | if args.cpu_change: 71 | cpu.change('userspace', cpu.min_freq()) 72 | cpu.dump() 73 | 74 | aio.set_event_loop(loop) 75 | 76 | if not args.endpoint: 77 | os.putenv('PYTHONPATH', 'src') 78 | server_fut = aio.create_subprocess_exec( 79 | 'python', 'benchmarks/japronto/micro.py', *args.server.split()) 80 | server = loop.run_until_complete(server_fut) 81 | os.unsetenv('PYTHONPATH') 82 | if not args.endpoint: 83 | process = psutil.Process(server.pid) 84 | elif args.pid: 85 | process = psutil.Process(args.pid) 86 | else: 87 | process = None 88 | 89 | cpu_p = 100 90 | while cpu_p > 5: 91 | cpu_p = psutil.cpu_percent(interval=1) 92 | print('CPU usage in 1 sec:', cpu_p) 93 | 94 | results = [] 95 | cpu_usages = [] 96 | process_cpu_usages = [] 97 | mem_usages = [] 98 | conn_cnt = [] 99 | if process: 100 | cpu_usage(process) 101 | for _ in range(10): 102 | results.append(run_wrk(loop, args.endpoint)) 103 | cpu_usages.append(psutil.cpu_percent()) 104 | if process: 105 | process_cpu_usages.append(cpu_usage(process)) 106 | conn_cnt.append(connections(process)) 107 | mem_usages.append(round(memory(process), 2)) 108 | print('.', end='') 109 | sys.stdout.flush() 110 | 111 | if not args.endpoint: 112 | server.terminate() 113 | loop.run_until_complete(server.wait()) 114 | 115 | if args.cpu_change: 116 | cpu.change('ondemand') 117 | 118 | print() 119 | print('RPS', results) 120 | print('Mem', mem_usages) 121 | print('Conn', conn_cnt) 122 | print('Server', process_cpu_usages) 123 | print('System', cpu_usages) 124 | median = statistics.median_grouped(results) 125 | stdev = round(statistics.stdev(results), 2) 126 | p = round((stdev / median) * 100, 2) 127 | print('median:', median, 'stdev:', stdev, '%', p) 128 | -------------------------------------------------------------------------------- /examples/1_hello/hello.py: -------------------------------------------------------------------------------- 1 | from japronto import Application 2 | 3 | 4 | # Views handle logic, take request as a parameter and 5 | # return the Response object back to the client 6 | def hello(request): 7 | return request.Response(text='Hello world!') 8 | 9 | 10 | # The Application instance is a fundamental concept. 11 | # It is a parent to all the resources and all the settings 12 | # can be tweaked there. 13 | app = Application() 14 | 15 | # The Router instance lets you register your handlers and execute 16 | # them depending on the url path and methods. 17 | app.router.add_route('/', hello) 18 | 19 | # Finally, start our server and handle requests until termination is 20 | # requested. Enabling debug lets you see request logs and stack traces. 21 | app.run(debug=True) 22 | -------------------------------------------------------------------------------- /examples/2_async/async.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from japronto import Application 3 | 4 | 5 | # This is a synchronous handler. 6 | def synchronous(request): 7 | return request.Response(text='I am synchronous!') 8 | 9 | 10 | # This is an asynchronous handler. It spends most of the time in the event loop. 11 | # It wakes up every second 1 to print and finally returns after 3 seconds. 12 | # This lets other handlers execute in the same processes while 13 | # from the point of view of the client it took 3 seconds to complete. 14 | async def asynchronous(request): 15 | for i in range(1, 4): 16 | await asyncio.sleep(1) 17 | print(i, 'seconds elapsed') 18 | 19 | return request.Response(text='3 seconds elapsed') 20 | 21 | 22 | app = Application() 23 | 24 | r = app.router 25 | r.add_route('/sync', synchronous) 26 | r.add_route('/async', asynchronous) 27 | 28 | app.run() 29 | -------------------------------------------------------------------------------- /examples/3_router/router.py: -------------------------------------------------------------------------------- 1 | from japronto import Application 2 | 3 | 4 | app = Application() 5 | r = app.router 6 | 7 | 8 | # Requests with the path set exactly to `/` and whatever method 9 | # will be directed here. 10 | def slash(request): 11 | return request.Response(text='Hello {} /!'.format(request.method)) 12 | 13 | 14 | r.add_route('/', slash) 15 | 16 | 17 | # Requests with the path set exactly to '/love' and the method 18 | # set exactly to `GET` will be directed here. 19 | def get_love(request): 20 | return request.Response(text='Got some love') 21 | 22 | 23 | r.add_route('/love', get_love, 'GET') 24 | 25 | 26 | # Requests with the path set exactly to '/methods' and the method 27 | # set to `POST` or `DELETE` will be directed here. 28 | def methods(request): 29 | return request.Response(text=request.method) 30 | 31 | 32 | r.add_route('/methods', methods, methods=['POST', 'DELETE']) 33 | 34 | 35 | # Requests with the path starting with `/params/` segment and followed 36 | # by two additional segments will be directed here. 37 | # Values of the additional segments will be stored inside `request.match_dict` 38 | # dictionary with keys taken from {} placeholders. A request to `/params/1/2` 39 | # would leave `match_dict` set to `{'p1': 1, 'p2': '2'}`. 40 | def params(request): 41 | return request.Response(text=str(request.match_dict)) 42 | 43 | 44 | r.add_route('/params/{p1}/{p2}', params) 45 | 46 | app.run() 47 | -------------------------------------------------------------------------------- /examples/4_request/request.py: -------------------------------------------------------------------------------- 1 | from json import JSONDecodeError 2 | 3 | from japronto import Application 4 | 5 | 6 | # Request line and headers. 7 | # This represents the part of a request that comes before message body. 8 | # Given an HTTP 1.1 `GET` request to `/basic?a=1` this would yield 9 | # `method` set to `GET`, `path` set to `/basic`, `version` set to `1.1` 10 | # `query_string` set to `a=1` and `query` set to `{'a': '1'}`. 11 | # Additionally if headers are sent they will be present in `request.headers` 12 | # dictionary. The keys are normalized to standard `Camel-Cased` convention. 13 | def basic(request): 14 | text = """Basic request properties: 15 | Method: {0.method} 16 | Path: {0.path} 17 | HTTP version: {0.version} 18 | Query string: {0.query_string} 19 | Query: {0.query}""".format(request) 20 | 21 | if request.headers: 22 | text += "\nHeaders:\n" 23 | for name, value in request.headers.items(): 24 | text += " {0}: {1}\n".format(name, value) 25 | 26 | return request.Response(text=text) 27 | 28 | 29 | # Message body 30 | # If there is a message body attached to a request (as in a case of `POST`) 31 | # the following attributes can be used to examine it. 32 | # Given a `POST` request with body set to `b'J\xc3\xa1'`, `Content-Length` header set 33 | # to `3` and `Content-Type` header set to `text/plain; charset=utf-8` this 34 | # would yield `mime_type` set to `'text/plain'`, `encoding` set to `'utf-8'`, 35 | # `body` set to `b'J\xc3\xa1'` and `text` set to `'Já'`. 36 | # `form` and `files` attributes are dictionaries respectively used for HTML forms and 37 | # HTML file uploads. The `json` helper property will try to decode `body` as a 38 | # JSON document and give you resulting Python data type. 39 | def body(request): 40 | text = """Body related properties: 41 | Mime type: {0.mime_type} 42 | Encoding: {0.encoding} 43 | Body: {0.body} 44 | Text: {0.text} 45 | Form parameters: {0.form} 46 | Files: {0.files} 47 | """.format(request) 48 | 49 | try: 50 | json = request.json 51 | except JSONDecodeError: 52 | pass 53 | else: 54 | text += "\nJSON:\n" 55 | text += str(json) 56 | 57 | return request.Response(text=text) 58 | 59 | 60 | # Miscellaneous 61 | # `route` will point to an instance of `Route` object representing 62 | # route chosen by router to handle this request. `hostname` and `port` 63 | # represent parsed `Host` header if any. `remote_addr` is the address of 64 | # a client or reverse proxy. If `keep_alive` is true the client requested to 65 | # keep the connection open after the response is delivered. `match_dict` contains 66 | # route placeholder values as documented in `2_router.md`. `cookies` contains 67 | # a dictionary of HTTP cookies if any. 68 | def misc(request): 69 | text = """Miscellaneous: 70 | Matched route: {0.route} 71 | Hostname: {0.hostname} 72 | Port: {0.port} 73 | Remote address: {0.remote_addr}, 74 | HTTP Keep alive: {0.keep_alive} 75 | Match parameters: {0.match_dict} 76 | """.strip().format(request) 77 | 78 | if request.cookies: 79 | text += "\nCookies:\n" 80 | for name, value in request.cookies.items(): 81 | text += " {0}: {1}\n".format(name, value) 82 | 83 | return request.Response(text=text) 84 | 85 | 86 | app = Application() 87 | app.router.add_route('/basic', basic) 88 | app.router.add_route('/body', body) 89 | app.router.add_route('/misc', misc) 90 | app.run() 91 | -------------------------------------------------------------------------------- /examples/5_response/response.py: -------------------------------------------------------------------------------- 1 | import random 2 | from http.cookies import SimpleCookie 3 | 4 | from japronto.app import Application 5 | 6 | 7 | # Providing just a text argument yields a `text/plain` response 8 | # encoded with `utf8` codec (charset set accordingly) 9 | def text(request): 10 | return request.Response(text='Hello world!') 11 | 12 | 13 | # You can override encoding by providing the `encoding` attribute. 14 | def encoding(request): 15 | return request.Response(text='Já pronto!', encoding='iso-8859-1') 16 | 17 | 18 | # You can also set a custom MIME type. 19 | def mime(request): 20 | return request.Response( 21 | mime_type="image/svg+xml", 22 | text=""" 23 | 24 | 25 | 26 | """) 27 | 28 | 29 | # Or serve binary data. `Content-Type` is set to `application/octet-stream` 30 | # automatically but you can always provide your own `mime_type`. 31 | def body(request): 32 | return request.Response(body=b'\xde\xad\xbe\xef') 33 | 34 | 35 | # There exist a shortcut `json` argument. This automatically encodes the 36 | # provided object as JSON and servers it with `Content-Type` set to 37 | # `application/json; charset=utf8` 38 | def json(request): 39 | return request.Response(json={'hello': 'world'}) 40 | 41 | 42 | # You can change the default 200 status `code` for another 43 | def code(request): 44 | return request.Response(code=random.choice([200, 201, 400, 404, 500])) 45 | 46 | 47 | # And of course you can provide custom `headers`. 48 | def headers(request): 49 | return request.Response( 50 | text='headers', 51 | headers={'X-Header': 'Value', 52 | 'Refresh': '5; url=https://xkcd.com/353/'}) 53 | 54 | 55 | # Or `cookies` by using Python standard library `http.cookies.SimpleCookie`. 56 | def cookies(request): 57 | cookies = SimpleCookie() 58 | cookies['hello'] = 'world' 59 | cookies['hello']['domain'] = 'localhost' 60 | cookies['hello']['path'] = '/' 61 | cookies['hello']['max-age'] = 3600 62 | cookies['city'] = 'São Paulo' 63 | 64 | return request.Response(text='cookies', cookies=cookies) 65 | 66 | 67 | app = Application() 68 | router = app.router 69 | router.add_route('/text', text) 70 | router.add_route('/encoding', encoding) 71 | router.add_route('/mime', mime) 72 | router.add_route('/body', body) 73 | router.add_route('/json', json) 74 | router.add_route('/code', code) 75 | router.add_route('/headers', headers) 76 | router.add_route('/cookies', cookies) 77 | app.run() 78 | -------------------------------------------------------------------------------- /examples/6_exceptions/exceptions.py: -------------------------------------------------------------------------------- 1 | from japronto import Application, RouteNotFoundException 2 | 3 | 4 | # These are our custom exceptions we want to turn into 200 response. 5 | class KittyError(Exception): 6 | def __init__(self): 7 | self.greet = 'meow' 8 | 9 | 10 | class DoggieError(Exception): 11 | def __init__(self): 12 | self.greet = 'woof' 13 | 14 | 15 | # The two handlers below raise exceptions which will be turned 16 | # into 200 responses by the handlers registered later 17 | def cat(request): 18 | raise KittyError() 19 | 20 | 21 | def dog(request): 22 | raise DoggieError() 23 | 24 | 25 | # This handler raises ZeroDivisionError which doesn't have an error 26 | # handler registered so it will result in 500 Internal Server Error 27 | def unhandled(request): 28 | 1 / 0 29 | 30 | 31 | app = Application() 32 | 33 | r = app.router 34 | r.add_route('/cat', cat) 35 | r.add_route('/dog', dog) 36 | r.add_route('/unhandled', unhandled) 37 | 38 | 39 | # These two are handlers for `Kitty` and `DoggyError`s. 40 | def handle_cat(request, exception): 41 | return request.Response(text='Just a kitty, ' + exception.greet) 42 | 43 | 44 | def handle_dog(request, exception): 45 | return request.Response(text='Just a doggie, ' + exception.greet) 46 | 47 | 48 | # You can also override default 404 handler if you want 49 | def handle_not_found(request, exception): 50 | return request.Response(code=404, text="Are you lost, pal?") 51 | 52 | 53 | # register all the error handlers so they are actually effective 54 | app.add_error_handler(KittyError, handle_cat) 55 | app.add_error_handler(DoggieError, handle_dog) 56 | app.add_error_handler(RouteNotFoundException, handle_not_found) 57 | 58 | app.run() 59 | -------------------------------------------------------------------------------- /examples/7_extend/extend.py: -------------------------------------------------------------------------------- 1 | from japronto import Application 2 | 3 | 4 | # This view accesses custom method host_startswith 5 | # and a custom property reversed_agent. Both are registered later. 6 | def extended_hello(request): 7 | if request.host_startswith('api.'): 8 | text = 'Hello ' + request.reversed_agent 9 | else: 10 | text = 'Hello stranger' 11 | 12 | return request.Response(text=text) 13 | 14 | 15 | # This view registers a callback, such callbacks are executed after handler 16 | # exits and the response is ready to be sent over the wire. 17 | def with_callback(request): 18 | def cb(r): 19 | print('Done!') 20 | 21 | request.add_done_callback(cb) 22 | 23 | return request.Response(text='cb') 24 | 25 | 26 | # This is a body for reversed_agent property 27 | def reversed_agent(request): 28 | return request.headers['User-Agent'][::-1] 29 | 30 | 31 | # This is a body for host_startswith method 32 | # Custom methods and properties always accept request 33 | # object. 34 | def host_startswith(request, prefix): 35 | return request.headers['Host'].startswith(prefix) 36 | 37 | 38 | app = Application() 39 | # Finally register the custom property and method 40 | # By default the names are taken from function names 41 | # unelss you provide `name` keyword parameter. 42 | app.extend_request(reversed_agent, property=True) 43 | app.extend_request(host_startswith) 44 | 45 | r = app.router 46 | r.add_route('/', extended_hello) 47 | r.add_route('/callback', with_callback) 48 | 49 | 50 | app.run() 51 | -------------------------------------------------------------------------------- /examples/8_template/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | japronto 8 | 9 | 10 | 11 | 12 |

Hello World!

13 | 14 |

Behold, the power of japronto!

15 | 16 | 17 | -------------------------------------------------------------------------------- /examples/8_template/template.py: -------------------------------------------------------------------------------- 1 | # examples/8_template/template.py 2 | from japronto import Application 3 | from jinja2 import Template 4 | 5 | 6 | # A view can read HTML from a file 7 | def index(request): 8 | with open('index.html') as html_file: 9 | return request.Response(text=html_file.read(), mime_type='text/html') 10 | 11 | 12 | # A view could also return a raw HTML string 13 | def example(request): 14 | return request.Response(text='

Some HTML!

', mime_type='text/html') 15 | 16 | 17 | template = Template('

Hello {{ name }}!

') 18 | 19 | # A view could also return a rendered jinja2 template 20 | def jinja(request): 21 | return request.Response(text=template.render(name='World'), 22 | mime_type='text/html') 23 | 24 | 25 | # Create the japronto application 26 | app = Application() 27 | 28 | # Add routes to the app 29 | app.router.add_route('/', index) 30 | app.router.add_route('/example', example) 31 | app.router.add_route('/jinja2', jinja) 32 | 33 | # Start the server 34 | app.run(debug=True) 35 | 36 | -------------------------------------------------------------------------------- /examples/todo_api/.gitignore: -------------------------------------------------------------------------------- 1 | todo.sqlite 2 | -------------------------------------------------------------------------------- /examples/todo_api/todo_api.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import sqlite3 3 | from functools import partial 4 | 5 | from japronto import Application 6 | 7 | 8 | def add_todo(request): 9 | cur = request.cursor 10 | todo = request.json["todo"] 11 | cur.execute("""INSERT INTO todos (todo) VALUES (?)""", (todo,)) 12 | last_id = cur.lastrowid 13 | cur.connection.commit() 14 | 15 | return request.Response(json={"id": last_id, "todo": todo}) 16 | 17 | 18 | def list_todos(request): 19 | cur = request.cursor 20 | cur.execute("""SELECT id, todo FROM todos""") 21 | todos = [{"id": id, "todo": todo} for id, todo in cur] 22 | 23 | return request.Response(json={"results": todos}) 24 | 25 | 26 | def show_todo(request): 27 | cur = request.cursor 28 | id = int(request.match_dict['id']) 29 | cur.execute("""SELECT id, todo FROM todos WHERE id = ?""", (id,)) 30 | todo = cur.fetchone() 31 | if not todo: 32 | return request.Response(code=404, json={"error": "not found"}) 33 | todo = {"id": todo[0], "todo": todo[1]} 34 | 35 | return request.Response(json=todo) 36 | 37 | 38 | def delete_todo(request): 39 | cur = request.cursor 40 | id = int(request.match_dict['id']) 41 | cur.execute("""DELETE FROM todos WHERE id = ?""", (id,)) 42 | if not cur.rowcount: 43 | return request.Response(code=404, json={"error": "not found"}) 44 | cur.connection.commit() 45 | 46 | return request.Response(json={}) 47 | 48 | 49 | DB_FILE = os.path.abspath( 50 | os.path.join(os.path.dirname(__file__), 'todo.sqlite')) 51 | db_connect = partial(sqlite3.connect, DB_FILE) 52 | 53 | 54 | def maybe_create_schema(): 55 | db = db_connect() 56 | db.execute(""" 57 | CREATE TABLE IF NOT EXISTS todos 58 | (id INTEGER PRIMARY KEY, todo TEXT)""") 59 | db.close() 60 | 61 | 62 | maybe_create_schema() 63 | app = Application() 64 | 65 | 66 | def cursor(request): 67 | def done_cb(request): 68 | request.extra['conn'].close() 69 | 70 | if 'conn' not in request.extra: 71 | request.extra['conn'] = db_connect() 72 | request.add_done_callback(done_cb) 73 | 74 | return request.extra['conn'].cursor() 75 | 76 | 77 | app.extend_request(cursor, property=True) 78 | router = app.router 79 | router.add_route('/todos', list_todos, method='GET') 80 | router.add_route('/todos/{id}', show_todo, method='GET') 81 | router.add_route('/todos/{id}', delete_todo, method='DELETE') 82 | router.add_route('/todos', add_todo, method='POST') 83 | 84 | app.run() 85 | -------------------------------------------------------------------------------- /integration_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/integration_tests/__init__.py -------------------------------------------------------------------------------- /integration_tests/common.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import ctypes.util 4 | import sys 5 | import time 6 | 7 | import psutil 8 | 9 | 10 | def start_server(script, *, stdout=None, path=None, sanitize=True, wait=True, 11 | return_process=False, buffer=False): 12 | if not isinstance(script, list): 13 | script = [script] 14 | if path: 15 | os.putenv('PYTHONPATH', path) 16 | if sanitize: 17 | os.putenv('LD_PRELOAD', ctypes.util.find_library('asan')) 18 | os.putenv('LSAN_OPTIONS', 'suppressions=misc/suppr.txt') 19 | if not buffer: 20 | os.putenv('PYTHONUNBUFFERED', '1') 21 | server = subprocess.Popen([sys.executable, *script], stdout=stdout) 22 | if not buffer: 23 | os.unsetenv('PYTHONUNBUFFERED') 24 | if sanitize: 25 | os.unsetenv('LSAN_OPTIONS') 26 | os.unsetenv('LD_PRELOAD') 27 | if path: 28 | os.unsetenv('PYTHONPATH') 29 | 30 | process = psutil.Process(server.pid) 31 | if wait: 32 | # wait until the server socket is open 33 | while 1: 34 | assert server.poll() is None 35 | conn_num = len(process.connections()) 36 | for child in process.children(): 37 | conn_num += len(child.connections()) 38 | if conn_num: 39 | break 40 | time.sleep(.001) 41 | 42 | assert server.poll() is None 43 | 44 | if return_process: 45 | return server, process 46 | else: 47 | return server 48 | -------------------------------------------------------------------------------- /integration_tests/drain.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from japronto.app import Application 4 | 5 | 6 | def slash(request): 7 | return request.Response() 8 | 9 | 10 | async def sleep(request): 11 | await asyncio.sleep(int(request.match_dict['sleep'])) 12 | return request.Response() 13 | 14 | 15 | app = Application() 16 | 17 | r = app.router 18 | r.add_route('/', slash) 19 | r.add_route('/sleep/{sleep}', sleep) 20 | 21 | 22 | if __name__ == '__main__': 23 | app.run() 24 | -------------------------------------------------------------------------------- /integration_tests/dump.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import asyncio 3 | 4 | 5 | from japronto.app import Application 6 | 7 | 8 | class ForcedException(Exception): 9 | pass 10 | 11 | 12 | def dump(request, exception=None): 13 | if not exception and 'Force-Raise' in request.headers: 14 | raise ForcedException(request.headers['Force-Raise']) 15 | 16 | body = request.body 17 | if body is not None: 18 | body = base64.b64encode(body).decode('ascii') 19 | 20 | result = { 21 | "method": request.method, 22 | "path": request.path, 23 | "query_string": request.query_string, 24 | "headers": request.headers, 25 | "match_dict": request.match_dict, 26 | "body": body, 27 | "route": request.route and request.route.pattern 28 | } 29 | 30 | if exception: 31 | result['exception'] = { 32 | "type": type(exception).__name__, 33 | "args": ", ".join(str(a) for a in exception.args) 34 | } 35 | 36 | return request.Response(code=500 if exception else 200, json=result) 37 | 38 | 39 | async def adump(request): 40 | sleep = float(request.query.get('sleep', 0)) 41 | await asyncio.sleep(sleep) 42 | 43 | return dump(request) 44 | 45 | 46 | app = Application() 47 | 48 | r = app.router 49 | r.add_route('/dump/{p1}/{p2}', dump) 50 | r.add_route('/dump1/{p1}/{p2}', dump) 51 | r.add_route('/dump2/{p1}/{p2}', dump) 52 | r.add_route('/async/dump/{p1}/{p2}', adump) 53 | r.add_route('/async/dump1/{p1}/{p2}', adump) 54 | r.add_route('/async/dump2/{p1}/{p2}', adump) 55 | app.add_error_handler(None, dump) 56 | 57 | 58 | if __name__ == '__main__': 59 | app.run() 60 | -------------------------------------------------------------------------------- /integration_tests/experiments.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture(autouse=True) 5 | def my_fix(request): 6 | print('auto') 7 | pytest.set_trace() 8 | 9 | 10 | @pytest.fixture(params=[1, 2]) 11 | def size_k(request): 12 | return request.param 13 | 14 | # @pytest.fixture(autouse=True, scope='module') 15 | # def a(): 16 | # print('a') 17 | 18 | 19 | # @pytest.fixture(scope='function', params=[1,2]) 20 | # def fix(): 21 | # print('> fix') 22 | # yield 3 23 | # print('< fix') 24 | 25 | 26 | # def test(my_fix): 27 | # print(test) 28 | 29 | def test1(size_k): 30 | print(test1) 31 | -------------------------------------------------------------------------------- /integration_tests/generators.py: -------------------------------------------------------------------------------- 1 | from integration_tests import strategies as st 2 | 3 | from hypothesis.strategies import SearchStrategy 4 | 5 | 6 | def generate_body(body, size_k): 7 | if size_k and body: 8 | if isinstance(body, list): 9 | length = sum(len(b) for b in body) 10 | else: 11 | length = len(body) 12 | body = body * ((size_k * 1024) // length + 1) 13 | 14 | return body 15 | 16 | 17 | def makeval(v, default_st, default=None): 18 | if isinstance(v, SearchStrategy): 19 | return v.example() 20 | 21 | if v is True: 22 | return default_st.example() 23 | 24 | if v is not None: 25 | return v 26 | 27 | return default 28 | 29 | 30 | def print_request(request): 31 | body = request['body'] 32 | if body: 33 | if isinstance(body, list): 34 | body = '{} chunks'.format(len(body)) 35 | else: 36 | if len(body) > 32: 37 | body = body[:32] + b'...' 38 | print(repr(request['method']), repr(request['path']), 39 | repr(request['query_string']), body) 40 | 41 | 42 | def generate_request(*, method=None, path=None, query_string=None, 43 | headers=None, body=None, size_k=None): 44 | request = {} 45 | request['method'] = makeval(method, st.method, 'GET') 46 | request['path'] = makeval(path, st.path, '/') 47 | request['query_string'] = makeval(query_string, st.query_string) 48 | request['headers'] = makeval(headers, st.headers) 49 | request['body'] = generate_body(makeval(body, st.body), size_k) 50 | 51 | return request 52 | 53 | 54 | def generate_combinations(reverse=False): 55 | props = ['method', 'path', 'query_string', 'headers', 'body'] 56 | sizes = [None, 8, 32, 64] 57 | if reverse: 58 | props = reversed(props) 59 | sizes = reversed(sizes) 60 | for prop in props: 61 | if prop == 'body': 62 | for size_k in sizes: 63 | yield {'body': True, 'size_k': size_k} 64 | else: 65 | yield {prop: True} 66 | 67 | 68 | def send_requests(conn, number, **kwargs): 69 | for _ in range(number): 70 | request = generate_request(**kwargs) 71 | print_request(request) 72 | conn.request(**request) 73 | conn.getresponse() 74 | -------------------------------------------------------------------------------- /integration_tests/longrun.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | import signal 4 | import atexit 5 | import os 6 | import time 7 | 8 | sys.path.insert(0, '.') 9 | 10 | import integration_tests.common # noqa 11 | import integration_tests.generators # noqa 12 | 13 | from misc import client # noqa 14 | 15 | 16 | def setup(): 17 | subprocess.check_call([ 18 | sys.executable, 'build.py', '--dest', '.test/longrun', 19 | '--kit', 'platform', '--disable-response-cache']) 20 | 21 | os.putenv('MALLOC_TRIM_THRESHOLD_', '0') 22 | server = integration_tests.common.start_server( 23 | 'integration_tests/dump.py', path='.test/longrun', sanitize=False) 24 | os.unsetenv('MALLOC_TRIM_THRESHOLD_') 25 | 26 | os.makedirs('.collector', exist_ok=True) 27 | 28 | os.putenv('COLLECTOR_FILE', '.collector/{}.json'.format(server.pid)) 29 | collector = subprocess.Popen([ 30 | sys.executable, 'misc/collector.py', str(server.pid)]) 31 | os.unsetenv('COLLECTOR_FILE') 32 | 33 | def cleanup(*args): 34 | try: 35 | server.terminate() 36 | assert server.wait() == 0 37 | finally: 38 | atexit.unregister(cleanup) 39 | 40 | atexit.register(cleanup) 41 | signal.signal(signal.SIGINT, cleanup) 42 | 43 | 44 | def run(): 45 | time.sleep(2) 46 | for reverse in [True, False]: 47 | for combination in integration_tests.generators.generate_combinations( 48 | reverse=reverse): 49 | conn = client.Connection('localhost:8080') 50 | time.sleep(2) 51 | integration_tests.generators.send_requests( 52 | conn, 200, **combination) 53 | time.sleep(2) 54 | conn.close() 55 | time.sleep(2) 56 | 57 | 58 | def main(): 59 | setup() 60 | run() 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /integration_tests/noleak.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | from japronto.app import Application 5 | 6 | 7 | prop = sys.argv[1] 8 | 9 | if prop == 'method': 10 | def noleak(request): 11 | return request.Response(text=request.method) 12 | elif prop == 'path': 13 | def noleak(request): 14 | return request.Response(text=request.path) 15 | elif prop == 'match_dict': 16 | def noleak(request): 17 | return request.Response(json=request.match_dict) 18 | elif prop == 'query_string': 19 | def noleak(request): 20 | return request.Response(text=request.query_string) 21 | elif prop == 'headers': 22 | def noleak(request): 23 | return request.Response(json=request.headers) 24 | elif prop == 'body': 25 | def noleak(request): 26 | return request.Response(body=request.body) 27 | elif prop == 'keep_alive': 28 | def noleak(request): 29 | return request.Response(text=str(request.keep_alive)) 30 | elif prop == 'route': 31 | def noleak(request): 32 | return request.Response(text=str(request.route)) 33 | 34 | app = Application() 35 | 36 | r = app.router 37 | r.add_route('/noleak/{p1}/{p2}', noleak) 38 | 39 | 40 | if __name__ == '__main__': 41 | app.run() 42 | -------------------------------------------------------------------------------- /integration_tests/reaper.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from japronto.app import Application 4 | 5 | reaper_settings = { 6 | 'check_interval': int(sys.argv[1]), 7 | 'idle_timeout': int(sys.argv[2]) 8 | } 9 | 10 | app = Application(reaper_settings=reaper_settings) 11 | 12 | if __name__ == '__main__': 13 | app.run() 14 | -------------------------------------------------------------------------------- /integration_tests/strategies.py: -------------------------------------------------------------------------------- 1 | import string 2 | import re 3 | 4 | from hypothesis import strategies as st 5 | sampled_from = st.sampled_from 6 | fixed_dictionaries = st.fixed_dictionaries 7 | lists = st.lists 8 | builds = st.builds 9 | integers = st.integers 10 | 11 | _method_alphabet = ''.join(chr(x) for x in range(33, 256) if x != 127) 12 | method = st.text(_method_alphabet, min_size=1) 13 | 14 | 15 | _path_alphabet = st.characters( 16 | blacklist_characters='?', blacklist_categories=['Cs']) 17 | path = st.text(_path_alphabet).map(lambda x: '/' + x) 18 | 19 | _param_alphabet = st.characters( 20 | blacklist_characters='/?', blacklist_categories=['Cs']) 21 | param = st.text(_param_alphabet, min_size=1) 22 | 23 | query_string = st.one_of(st.text(), st.none()) 24 | 25 | _name_alphabet = string.digits + string.ascii_letters + '!#$%&\'*+-.^_`|~' 26 | _names = st.text(_name_alphabet, min_size=1).map(lambda x: 'X-' + x) 27 | _value_alphabet = ''.join(chr(x) for x in range(ord(' '), 256) if x != 127) 28 | _is_illegal_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search 29 | _values = st.text(_value_alphabet, min_size=1) \ 30 | .filter(lambda x: not _is_illegal_value(x)).map(lambda x: x.strip()) 31 | headers = st.lists(st.tuples(_names, _values), max_size=48) 32 | 33 | identity_body = st.one_of(st.binary(), st.none()) 34 | chunked_body = st.lists(st.binary(min_size=24)) 35 | body = st.one_of(st.binary(), st.none(), chunked_body) 36 | -------------------------------------------------------------------------------- /integration_tests/test_drain.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import subprocess 3 | import time 4 | 5 | from misc import client 6 | import integration_tests.common 7 | 8 | 9 | pytestmark = pytest.mark.needs_build 10 | 11 | 12 | @pytest.fixture(scope='function') 13 | def server(): 14 | return integration_tests.common.start_server( 15 | 'integration_tests/drain.py', stdout=subprocess.PIPE, path='.test') 16 | 17 | 18 | @pytest.fixture(scope='function') 19 | def server_terminate(server): 20 | def terminate(): 21 | server.terminate() 22 | assert server.wait() == 0 23 | 24 | stdout = server.stdout.read() 25 | 26 | return [l.decode('utf-8').strip() for l in stdout.splitlines()] 27 | 28 | yield terminate 29 | 30 | server.terminate() 31 | 32 | 33 | @pytest.fixture(scope='function') 34 | def connect(): 35 | connections = [] 36 | 37 | def _connect(): 38 | conn = client.Connection('localhost:8080') 39 | conn.maybe_connect() 40 | 41 | connections.append(conn) 42 | 43 | return conn 44 | 45 | yield _connect 46 | 47 | for c in connections: 48 | c.close() 49 | 50 | 51 | def test_no_connections(server_terminate): 52 | lines = server_terminate() 53 | 54 | assert lines[-1] == 'Termination request received' 55 | 56 | 57 | @pytest.mark.parametrize('num', range(1, 5)) 58 | def test_unclosed_connections(num, connect, server_terminate): 59 | for _ in range(num): 60 | connect() 61 | 62 | lines = server_terminate() 63 | 64 | assert lines[-1] == '{} idle connections closed immediately'.format(num) 65 | 66 | 67 | @pytest.mark.parametrize('num', range(1, 5)) 68 | def test_closed_connections(num, connect, server_terminate): 69 | for _ in range(num): 70 | con = connect() 71 | con.close() 72 | 73 | lines = server_terminate() 74 | 75 | assert lines[-1] == 'Termination request received' 76 | 77 | 78 | @pytest.mark.parametrize('num', range(1, 5)) 79 | def test_unclosed_requests(num, connect, server_terminate): 80 | for _ in range(num): 81 | con = connect() 82 | con.putrequest('GET', '/') 83 | con.endheaders() 84 | 85 | lines = server_terminate() 86 | 87 | assert lines[-1] == '{} idle connections closed immediately'.format(num) 88 | 89 | 90 | @pytest.mark.parametrize('num', range(1, 5)) 91 | def test_closed_requests(num, connect, server_terminate): 92 | for _ in range(num): 93 | con = connect() 94 | con.putrequest('GET', '/') 95 | con.endheaders() 96 | con.getresponse() 97 | con.close() 98 | 99 | lines = server_terminate() 100 | 101 | assert lines[-1] == 'Termination request received' 102 | 103 | 104 | @pytest.mark.parametrize('num', range(1, 3)) 105 | def test_pipelined(num, connect, server_terminate): 106 | connections = [] 107 | 108 | for _ in range(num): 109 | con = connect() 110 | connections.append(con) 111 | con.putrequest('GET', '/sleep/1') 112 | con.endheaders() 113 | 114 | lines = server_terminate() 115 | 116 | assert '{} connections busy, read-end closed'.format(num) in lines 117 | assert not any(l.startswith('Forcefully killing') for l in lines) 118 | 119 | assert all(c.getresponse().status == 200 for c in connections) 120 | 121 | 122 | @pytest.mark.parametrize('num', range(1, 3)) 123 | def test_pipelined_timeout(num, connect, server_terminate): 124 | connections = [] 125 | 126 | for _ in range(num): 127 | con = connect() 128 | connections.append(con) 129 | con.putrequest('GET', '/sleep/10') 130 | con.endheaders() 131 | 132 | lines = server_terminate() 133 | 134 | assert '{} connections busy, read-end closed'.format(num) in lines 135 | assert 'Forcefully killing {} connections'.format(num) in lines 136 | 137 | assert all(c.getresponse().status == 503 for c in connections) 138 | 139 | 140 | def test_refuse(connect, server): 141 | con = connect() 142 | con.putrequest('GET', '/sleep/10') 143 | con.endheaders() 144 | 145 | server.terminate() 146 | 147 | # give time for the signal to propagate 148 | time.sleep(1) 149 | 150 | with pytest.raises(ConnectionRefusedError): 151 | con = connect() 152 | 153 | assert server.wait() == 0 154 | -------------------------------------------------------------------------------- /integration_tests/test_noleak.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from misc import client 4 | import integration_tests.common 5 | 6 | 7 | pytestmark = pytest.mark.needs_build( 8 | '--extra-compile=-DPROTOCOL_TRACK_REFCNT=1', dest='.test/noleak') 9 | 10 | 11 | @pytest.fixture(scope='function') 12 | def server(request): 13 | arg = request.node.get_marker('arg').args[0] 14 | 15 | server = integration_tests.common.start_server([ 16 | 'integration_tests/noleak.py', arg], path='.test/noleak') 17 | 18 | yield server 19 | 20 | server.terminate() 21 | server.wait() == 0 22 | 23 | 24 | @pytest.fixture(scope='function') 25 | def connection(server): 26 | conn = client.Connection('localhost:8080') 27 | yield conn 28 | conn.close() 29 | 30 | 31 | @pytest.mark.arg('method') 32 | def test_method(connection): 33 | methods = ['GET', 'POST', 'PATCH', 'DELETE', 'PUT'] 34 | 35 | for method in methods: 36 | connection.putrequest(method, '/noleak/1/2') 37 | connection.endheaders() 38 | 39 | response = connection.getresponse() 40 | assert response.status == 200 41 | 42 | 43 | @pytest.mark.arg('path') 44 | def test_path(connection): 45 | paths = ['/noleak/1/2', '/noleak/3/4', '/noleak/5/4', '/noleak/6/7'] 46 | 47 | for path in paths: 48 | connection.putrequest('GET', path) 49 | connection.endheaders() 50 | 51 | response = connection.getresponse() 52 | assert response.status == 200 53 | 54 | 55 | @pytest.mark.arg('match_dict') 56 | def test_match_dict(connection): 57 | paths = ['/noleak/1/2', '/noleak/3/4', '/noleak/5/4', '/noleak/6/7'] 58 | 59 | for path in paths: 60 | connection.putrequest('GET', path) 61 | connection.endheaders() 62 | 63 | response = connection.getresponse() 64 | assert response.status == 200 65 | 66 | 67 | @pytest.mark.arg('query_string') 68 | def test_query_string(connection): 69 | query_strings = ['?', None, '?a', None, '?', None, '?b', None, '?'] 70 | 71 | for query_string in query_strings: 72 | connection.putrequest('GET', '/noleak/1/2', query_string) 73 | connection.endheaders() 74 | 75 | response = connection.getresponse() 76 | assert response.status == 200 77 | 78 | 79 | @pytest.mark.arg('headers') 80 | def test_headers(connection): 81 | header_list = [{}, {"X-a": "b"}, {}, {"X-b": "c"}, {}, {"X-c": "d"}] 82 | 83 | for headers in header_list: 84 | connection.putrequest('GET', '/noleak/1/2') 85 | for name, value in headers.items(): 86 | connection.putheader(name, value) 87 | connection.endheaders() 88 | 89 | response = connection.getresponse() 90 | assert response.status == 200 91 | 92 | 93 | @pytest.mark.arg('body') 94 | def test_body(connection): 95 | bodies = [None, b'a', None, b'b', None, b'c', None, b'd', None, b'e'] 96 | 97 | for body in bodies: 98 | connection.putrequest('GET', '/noleak/1/2') 99 | if body: 100 | connection.putheader('Content-Length', str(len(body))) 101 | connection.endheaders(body) 102 | 103 | response = connection.getresponse() 104 | assert response.status == 200 105 | 106 | 107 | @pytest.mark.arg('keep_alive') 108 | def test_keep_alive(request): 109 | keep_alives = [True, False, True, True, False, False, True, False] 110 | 111 | still_open = False 112 | 113 | for keep_alive in keep_alives: 114 | if not still_open: 115 | connection_gen = connection(request.getfixturevalue('server')) 116 | conn = next(connection_gen) 117 | still_open = keep_alive 118 | conn.putrequest('GET', '/noleak/1/2') 119 | if not keep_alive: 120 | conn.putheader('Connection', 'close') 121 | conn.endheaders() 122 | 123 | response = conn.getresponse() 124 | assert response.status == 200 125 | 126 | 127 | @pytest.mark.arg('route') 128 | def test_route(connection): 129 | for _ in range(7): 130 | connection.putrequest('GET', '/noleak/1/2') 131 | connection.endheaders() 132 | 133 | response = connection.getresponse() 134 | assert response.status == 200 135 | -------------------------------------------------------------------------------- /integration_tests/test_perror.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from hypothesis import given, strategies as st, settings, Verbosity 3 | import subprocess 4 | import queue 5 | import threading 6 | from functools import partial 7 | 8 | from misc import client 9 | import integration_tests.common 10 | 11 | 12 | pytestmark = pytest.mark.needs_build 13 | 14 | 15 | @pytest.fixture(autouse=True, scope='module') 16 | def server(): 17 | server = integration_tests.common.start_server( 18 | ['-u', 'integration_tests/dump.py'], 19 | stdout=subprocess.PIPE, path='.test') 20 | 21 | accepting = server.stdout.readline().decode('utf-8') 22 | assert accepting.startswith('Accepting connections ') 23 | 24 | yield server 25 | 26 | server.terminate() 27 | server.wait() == 0 28 | 29 | 30 | @pytest.fixture 31 | def line_getter(server): 32 | q = queue.Queue() 33 | 34 | def enqueue_output(): 35 | q.put(server.stdout.readline().strip().decode('utf-8')) 36 | 37 | class LineGetter: 38 | def start(self): 39 | self.thread = threading.Thread(target=enqueue_output) 40 | self.thread.start() 41 | 42 | def wait(self): 43 | self.thread.join() 44 | return q.get() 45 | 46 | return LineGetter() 47 | 48 | 49 | @pytest.fixture() 50 | def connect(request): 51 | return partial(client.Connection, 'localhost:8080') 52 | 53 | 54 | full_request_line = 'GET /asd?qwe HTTP/1.1' 55 | 56 | 57 | def make_truncated_request_line(cut): 58 | return full_request_line[:-cut] 59 | 60 | 61 | st_request_cut = st.integers(min_value=1, max_value=len(full_request_line) - 1) 62 | st_request_line = st.builds(make_truncated_request_line, st_request_cut) 63 | 64 | 65 | @given(request_line=st_request_line) 66 | @settings(verbosity=Verbosity.verbose, max_examples=20) 67 | def test_truncated_request_line(line_getter, connect, request_line): 68 | connection = connect() 69 | line_getter.start() 70 | 71 | connection.putline(request_line) 72 | 73 | assert line_getter.wait() == 'malformed_headers' 74 | 75 | response = connection.getresponse() 76 | assert response.status == 400 77 | assert response.text == 'malformed_headers' 78 | 79 | 80 | @given(request_line=st_request_line) 81 | @settings(verbosity=Verbosity.verbose, max_examples=20) 82 | def test_truncated_request_line_disconnect(line_getter, connect, request_line): 83 | connection = connect() 84 | line_getter.start() 85 | 86 | connection.putclose(request_line) 87 | 88 | assert line_getter.wait() == 'incomplete_headers' 89 | 90 | 91 | full_header = 'X-Header: asd' 92 | 93 | 94 | def make_truncated_header(cut): 95 | return full_header[:-cut] 96 | 97 | 98 | st_header_cut = st.integers(min_value=5, max_value=len(full_header) - 1) 99 | st_header_line = st.builds(make_truncated_header, st_header_cut) 100 | 101 | 102 | @given(header_line=st_header_line) 103 | @settings(verbosity=Verbosity.verbose, max_examples=20) 104 | def test_truncated_header(line_getter, connect, header_line): 105 | connection = connect() 106 | line_getter.start() 107 | connection.putline(full_request_line) 108 | connection.putline(header_line) 109 | connection.putline() 110 | 111 | assert line_getter.wait() == 'malformed_headers' 112 | 113 | response = connection.getresponse() 114 | assert response.status == 400 115 | assert response.text == 'malformed_headers' 116 | 117 | 118 | @given(header_line=st_header_line) 119 | @settings(verbosity=Verbosity.verbose, max_examples=20) 120 | def test_truncated_header_disconnect(line_getter, connect, header_line): 121 | connection = connect() 122 | line_getter.start() 123 | connection.putline(full_request_line) 124 | connection.putclose(header_line) 125 | 126 | assert line_getter.wait() == 'incomplete_headers' 127 | 128 | 129 | @pytest.mark.parametrize('value', [ 130 | '', 131 | '+5', 132 | '-5', 133 | '0x12', 134 | '12a' 135 | ]) 136 | def test_invalid_content_length(line_getter, connect, value): 137 | connection = connect() 138 | line_getter.start() 139 | connection.putline(full_request_line) 140 | connection.putheader('Content-Length', value) 141 | connection.putline() 142 | 143 | assert line_getter.wait() == 'invalid_headers' 144 | 145 | response = connection.getresponse() 146 | assert response.status == 400 147 | assert response.text == 'invalid_headers' 148 | -------------------------------------------------------------------------------- /integration_tests/test_reaper.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | import time 3 | 4 | import pytest 5 | 6 | from misc import client 7 | import integration_tests.common 8 | 9 | 10 | pytestmark = pytest.mark.needs_build 11 | 12 | 13 | @pytest.fixture(scope='function', params=[2, 3, 4]) 14 | def get_connections_and_wait(request): 15 | server, process = integration_tests.common.start_server([ 16 | 'integration_tests/reaper.py', '1', str(request.param)], path='.test', 17 | return_process=True) 18 | 19 | def connection_num(): 20 | return len( 21 | set(c.fd for c in process.connections()) | 22 | set(c.fd for p in process.children() for c in p.connections())) 23 | 24 | yield connection_num, partial(time.sleep, request.param) 25 | 26 | server.terminate() 27 | assert server.wait() == 0 28 | 29 | 30 | def test_empty(get_connections_and_wait): 31 | get_connections, wait = get_connections_and_wait 32 | conn = client.Connection('localhost:8080') 33 | 34 | assert get_connections() == 1 35 | 36 | conn.maybe_connect() 37 | time.sleep(.1) 38 | 39 | assert get_connections() == 2 40 | 41 | wait() 42 | 43 | assert get_connections() == 1 44 | 45 | 46 | def test_request(get_connections_and_wait): 47 | get_connections, wait = get_connections_and_wait 48 | conn = client.Connection('localhost:8080') 49 | 50 | assert get_connections() == 1 51 | 52 | conn.putrequest('GET', '/') 53 | conn.endheaders() 54 | 55 | assert get_connections() == 2 56 | 57 | wait() 58 | time.sleep(1) 59 | 60 | assert get_connections() == 1 61 | -------------------------------------------------------------------------------- /misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/misc/__init__.py -------------------------------------------------------------------------------- /misc/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ~ 4 | 5 | sudo apt-get update 6 | sudo apt-get install -y python3 git libbz2-dev libz-dev libsqlite3-dev libssl-dev gcc make libffi-dev lcov 7 | git clone https://github.com/yyuu/pyenv .pyenv 8 | .pyenv/bin/pyenv install -v 3.6.0 9 | wget https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz 10 | tar xvfz virtualenv-15.1.0.tar.gz 11 | python3 virtualenv-15.1.0/virtualenv.py -p .pyenv/versions/3.6.0/bin/python japronto-env 12 | git clone https://github.com/squeaky-pl/japronto 13 | 14 | cd japronto/src/picohttpparser 15 | ./build 16 | cd - 17 | 18 | cd japronto 19 | ../japronto-env/bin/pip install -r requirements.txt 20 | ../japronto-env/bin/python build.py --kit=platform 21 | cd - 22 | 23 | git clone https://github.com/wg/wrk 24 | cd wrk 25 | make 26 | cd - 27 | 28 | cp wrk/wrk japronto 29 | -------------------------------------------------------------------------------- /misc/buggers.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import psutil 3 | 4 | noisy = ['atom', 'chrome', 'firefox', 'dropbox', 'opera', 'spotify', 5 | 'gnome-documents'] 6 | 7 | 8 | def silence(): 9 | for proc in psutil.process_iter(): 10 | if proc.name() in noisy: 11 | proc.suspend() 12 | 13 | def noise(): 14 | for proc in psutil.process_iter(): 15 | if proc.name() in noisy: 16 | proc.resume() 17 | atexit.register(noise) 18 | -------------------------------------------------------------------------------- /misc/cleanup_script.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def main(): 5 | fp = open(sys.argv[1]) 6 | 7 | for line in fp: 8 | line = line.rstrip() 9 | if line.startswith('\t'): 10 | rest = line[18:] 11 | name_addr, _, rest = rest.partition(' ') 12 | name, _, addr = name_addr.partition('+') 13 | line = line[:18] + name + ' ' + rest 14 | 15 | print(line) 16 | 17 | fp.close() 18 | 19 | 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /misc/client.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import urllib.parse 3 | import json 4 | 5 | 6 | def readline(sock): 7 | line = b'' 8 | while not line.endswith(b'\r\n'): 9 | line += sock.recv(1) 10 | 11 | return line 12 | 13 | 14 | def readexact(sock, size): 15 | data = b'' 16 | while size: 17 | chunk = sock.recv(size) 18 | data += chunk 19 | size -= len(chunk) 20 | 21 | return data 22 | 23 | 24 | class Response: 25 | def __init__(self, sock): 26 | self.sock = sock 27 | 28 | self.read_status_line() 29 | self.read_headers() 30 | self.read_body() 31 | 32 | def read_status_line(self): 33 | status_line = b'' 34 | while not status_line: 35 | status_line = readline(self.sock).strip() 36 | _, self.status, self.reason = status_line.split(None, 2) 37 | self.status = int(self.status) 38 | 39 | def read_headers(self): 40 | self.headers = {} 41 | 42 | while 1: 43 | line = readline(self.sock).strip() 44 | if not line: 45 | break 46 | 47 | name, value = line.split(b':') 48 | name = name.strip().decode('ascii').title() 49 | value = value.strip().decode('latin1') 50 | self.headers[name] = value 51 | 52 | @property 53 | def encoding(self): 54 | content_type = self.headers.get('Content-Type') 55 | if not content_type: 56 | return 'latin1' 57 | 58 | _, *rest = [v.split('=') for v in content_type.split(';')] 59 | 60 | rest = {k.strip(): v.strip() for k, v in rest} 61 | 62 | return rest.get('charset', 'iso-8859-1') 63 | 64 | def read_body(self): 65 | self.body = readexact(self.sock, int(self.headers['Content-Length'])) 66 | self.text = self.body.decode(self.encoding) 67 | 68 | @property 69 | def json(self): 70 | return json.loads(self.text) 71 | 72 | 73 | class Connection: 74 | def __init__(self, addr): 75 | self.addr = addr 76 | self.sock = None 77 | 78 | def maybe_connect(self): 79 | if self.sock: 80 | return self.sock 81 | 82 | addr = self.addr.split(':') 83 | addr[1] = int(addr[1]) 84 | addr = tuple(addr) 85 | 86 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 87 | self.sock.connect(addr) 88 | 89 | return self.sock 90 | 91 | def putline(self, line=None): 92 | line = line or b'' 93 | sock = self.maybe_connect() 94 | if not isinstance(line, bytes): 95 | line = str(line).encode('latin1') 96 | sock.sendall(line + b'\r\n') 97 | 98 | def putclose(self, data): 99 | sock = self.maybe_connect() 100 | if not isinstance(data, bytes): 101 | data = str(data).encode('latin1') 102 | sock.sendall(data) 103 | self.close() 104 | 105 | def putrequest(self, method, path, query_string=None): 106 | url = urllib.parse.quote(path) 107 | if query_string is not None: 108 | url += '?' + urllib.parse.quote(query_string) 109 | 110 | request_line = "{method} {url} HTTP/1.1" \ 111 | .format(method=method, url=url) 112 | self.putline(request_line) 113 | 114 | def request(self, method, path, query_string=None, headers=None, 115 | body=None): 116 | self.putrequest(method, path, query_string) 117 | headers = headers or [] 118 | for name, value in headers: 119 | self.putheader(name, value) 120 | if body is not None: 121 | if isinstance(body, list): 122 | self.putheader('Transfer-Encoding', 'chunked') 123 | else: 124 | self.putheader('Content-Length', str(len(body))) 125 | 126 | self.endheaders(body) 127 | 128 | def putheader(self, name, value): 129 | header_line = name + ': ' + value 130 | self.putline(header_line) 131 | 132 | def endheaders(self, body=None): 133 | self.putline() 134 | if body is not None: 135 | sock = self.maybe_connect() 136 | if isinstance(body, list): 137 | for chunk in chunked_encoder(body): 138 | sock.sendall(chunk) 139 | else: 140 | sock.sendall(body) 141 | 142 | def getresponse(self): 143 | return Response(self.sock) 144 | 145 | def close(self): 146 | self.sock.close() 147 | 148 | 149 | def chunked_encoder(data): 150 | for chunk in data: 151 | if not chunk: 152 | continue 153 | yield '{:X}\r\n'.format(len(chunk)).encode('ascii') 154 | yield chunk + b'\r\n' 155 | yield b'0\r\n\r\n' 156 | -------------------------------------------------------------------------------- /misc/collector.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | import sys 3 | import time 4 | import os 5 | import json 6 | from functools import partial 7 | 8 | 9 | def get_connections(process): 10 | return len( 11 | set(c.fd for c in process.connections()) | 12 | set(c.fd for p in process.children() for c in p.connections())) 13 | 14 | 15 | def get_memory(p): 16 | return p.memory_full_info().uss \ 17 | + sum(c.memory_full_info().uss for c in p.children()) 18 | 19 | 20 | def sample_process(pid): 21 | process = psutil.Process(pid) 22 | samples = [] 23 | 24 | while 1: 25 | try: 26 | uss = get_memory(process) 27 | conn = get_connections(process) 28 | except (psutil.NoSuchProcess, psutil.AccessDenied): 29 | break 30 | 31 | samples.append({ 32 | 't': time.monotonic(), 33 | 'uss': uss, 'conn': conn, 'type': 'proc'}) 34 | time.sleep(.5) 35 | 36 | return samples 37 | 38 | 39 | def main(): 40 | pid = int(sys.argv[1]) 41 | 42 | samples = sample_process(pid) 43 | 44 | with open(os.environ['COLLECTOR_FILE'], 'a') as fp: 45 | for sample in samples: 46 | fp.write(json.dumps(sample) + '\n') 47 | 48 | print('Collector info written to', os.environ['COLLECTOR_FILE']) 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /misc/cpu.py: -------------------------------------------------------------------------------- 1 | """ 2 | CPU file 3 | """ 4 | 5 | 6 | # module imports 7 | import subprocess 8 | 9 | 10 | # cpu location 11 | CPU_PREFIX = '/sys/devices/system/cpu/' 12 | 13 | 14 | def save(): 15 | """ 16 | save function 17 | """ 18 | results = {} 19 | cpu_number = 0 20 | 21 | while True: 22 | try: 23 | _file = open( 24 | CPU_PREFIX + 'cpu{}/cpufreq/scaling_governor'.format(cpu_number)) 25 | except: 26 | break 27 | 28 | governor = _file.read().strip() 29 | results.setdefault(cpu_number, {})['governor'] = governor 30 | 31 | _file.close() 32 | 33 | try: 34 | _file = open( 35 | CPU_PREFIX + 'cpu{}/cpufreq/scaling_cur_freq'.format(cpu_number)) 36 | except: 37 | break 38 | 39 | results[cpu_number]['freq'] = _file.read().strip() 40 | 41 | _file.close() 42 | 43 | cpu_number += 1 44 | 45 | return results 46 | 47 | 48 | def change(governor, freq=None): 49 | """ 50 | change function 51 | """ 52 | cpu_number = 0 53 | 54 | while True: 55 | try: 56 | subprocess.check_output([ 57 | "sudo", "bash", "-c", 58 | "echo {governor} > {CPU_PREFIX}cpu{cpu_number}/cpufreq/scaling_governor" 59 | .format(governor=governor, 60 | CPU_PREFIX=CPU_PREFIX, 61 | cpu_number=cpu_number)], 62 | stderr=subprocess.STDOUT) 63 | except: 64 | break 65 | 66 | if freq: 67 | subprocess.check_output([ 68 | "sudo", "bash", "-c", 69 | "echo {freq} > {CPU_PREFIX}cpu{cpu_number}/cpufreq/scaling_setspeed" 70 | .format(freq=freq, 71 | CPU_PREFIX=CPU_PREFIX, 72 | cpu_number=cpu_number)], 73 | stderr=subprocess.STDOUT) 74 | 75 | cpu_number += 1 76 | 77 | 78 | def available_freq(): 79 | """ 80 | function for checking available frequency 81 | """ 82 | _file = open(CPU_PREFIX + 'cpu0/cpufreq/scaling_available_frequencies') 83 | 84 | freq = [int(_file) for _file in _file.read().strip().split()] 85 | 86 | _file.close() 87 | 88 | return freq 89 | 90 | 91 | def min_freq(): 92 | """ 93 | function for returning minimum available frequency 94 | """ 95 | return min(available_freq()) 96 | 97 | 98 | def max_freq(): 99 | """ 100 | function for returning maximum avaliable frequency 101 | """ 102 | return max(available_freq()) 103 | 104 | 105 | def dump(): 106 | """ 107 | dump function 108 | """ 109 | 110 | try: 111 | sensors = subprocess.check_output('sensors').decode('utf-8') 112 | 113 | except (FileNotFoundError, subprocess.CalledProcessError): 114 | print("Couldn't read CPU temp") 115 | 116 | else: 117 | cores = [] 118 | 119 | for line in sensors.splitlines(): 120 | if line.startswith('Core '): 121 | core, rest = line.split(':') 122 | temp = rest.strip().split()[0] 123 | cores.append((core, temp)) 124 | 125 | for core, temp in cores: 126 | print(core + ':', temp) 127 | 128 | cpu_number = 0 129 | 130 | while True: 131 | try: 132 | _file = open( 133 | CPU_PREFIX + 'cpu{}/cpufreq/scaling_governor'.format(cpu_number)) 134 | except: 135 | break 136 | 137 | print('Core ' + str(cpu_number) + ':', _file.read().strip(), end=', ') 138 | 139 | _file.close() 140 | 141 | try: 142 | _file = open( 143 | CPU_PREFIX + 'cpu{}/cpufreq/scaling_cur_freq'.format(cpu_number)) 144 | except: 145 | break 146 | 147 | freq = round(int(_file.read()) / 10 ** 6, 2) 148 | 149 | print(freq, 'GHz') 150 | 151 | cpu_number += 1 152 | -------------------------------------------------------------------------------- /misc/do_perf.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import sys 4 | import argparse 5 | 6 | import parsers 7 | import cases 8 | import buggers 9 | import cpu 10 | 11 | 12 | def get_http10long(): 13 | return cases.base['10long'].data 14 | 15 | 16 | def get_websites(size=2 ** 18): 17 | data = b'' 18 | while len(data) < size: 19 | for c in cases.websites.values(): 20 | data += c.data 21 | 22 | return data 23 | 24 | 25 | if __name__ == '__main__': 26 | print('pid', os.getpid()) 27 | 28 | cpu.dump() 29 | buggers.silence() 30 | 31 | argparser = argparse.ArgumentParser(description='do_perf') 32 | argparser.add_argument( 33 | '-p', '--parsers', dest='parsers', default='cffi,cext') 34 | argparser.add_argument( 35 | '-b', '--benchmarks', dest='benchmarks', 36 | default='http10long,websites,websitesn') 37 | 38 | result = argparser.parse_args(sys.argv[1:]) 39 | parsers = result.parsers.split(',') 40 | benchmarks = result.benchmarks.split(',') 41 | 42 | one_shot = [b for b in benchmarks if b in ['http10long', 'websites']] 43 | multi_shot = [b for b in benchmarks if b in ['websitesn']] 44 | 45 | setup = """ 46 | import parsers 47 | import do_perf 48 | parser, _ = parsers.make_{}(parsers.NullProtocol) 49 | data = do_perf.get_{}() 50 | """ 51 | 52 | loop = """ 53 | parser.feed(data) 54 | parser.feed_disconnect() 55 | """ 56 | 57 | for dataset in one_shot: 58 | for parser in parsers: 59 | print('-- {} {} --'.format(dataset, parser)) 60 | subprocess.check_call([ 61 | 'python', '-m', 'perf', 'timeit', '-s', 62 | setup.format(parser, dataset), loop]) 63 | print() 64 | 65 | setup += """ 66 | import parts 67 | p = parts.make_parts(data, parts.fancy_series(1450)) 68 | """ 69 | 70 | loop = """ 71 | for i in p: 72 | parser.feed(i) 73 | parser.feed_disconnect() 74 | """ 75 | 76 | if multi_shot: 77 | for parser in parsers: 78 | print('-- website parts {} --'.format(parser)) 79 | subprocess.check_call([ 80 | 'python', '-m', 'perf', 'timeit', '-s', 81 | setup.format(parser, 'websites'), loop]) 82 | print() 83 | -------------------------------------------------------------------------------- /misc/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6.0-slim 2 | 3 | RUN pip3 install japronto 4 | ENV PYTHONUNBUFFERED=1 5 | ENTRYPOINT ["japronto"] 6 | -------------------------------------------------------------------------------- /misc/parts.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | import types 3 | import math 4 | 5 | 6 | def make_parts(value, get_size, dir=1): 7 | parts = [] 8 | 9 | left = bytearray(value) 10 | while left: 11 | if isinstance(get_size, types.GeneratorType): 12 | size = next(get_size) 13 | else: 14 | size = get_size 15 | 16 | if dir == 1: 17 | parts.append(bytes(left[:size])) 18 | left = left[size:] 19 | else: 20 | parts.append(bytes(left[-size:])) 21 | left = left[:-size] 22 | 23 | return parts if dir == 1 else list(reversed(parts)) 24 | 25 | 26 | def one_part(value): 27 | return [value] 28 | 29 | 30 | def geometric_series(): 31 | s = 2 32 | while 1: 33 | yield s 34 | s *= 2 35 | 36 | 37 | def fancy_series(minimum=2): 38 | x = 0 39 | while 1: 40 | yield int(minimum + abs(math.sin(x / 3)) * 64) 41 | x += 1 42 | -------------------------------------------------------------------------------- /misc/perf.md: -------------------------------------------------------------------------------- 1 | Capturing performance data with perf 2 | ==================================== 3 | 4 | For best results the C source should be built with `-g -O0`. 5 | 6 | Run the server, record performance events with `-F` frequency `997 Hz`, `-a` all processes and `-g` take stack info. 7 | 8 | ``` 9 | python examples/simple/simple.py -p c & \ 10 | perf record -F 997 -a -g -- sleep 59 & \ 11 | sleep 1 && ./wrk -c 100 -t 1 -d 60 http://localhost:8080 & 12 | ``` 13 | 14 | Write data to a text file filtering for pid `2836` (should be server process) 15 | 16 | ``` 17 | perf script --pid 2836 > out.perf 18 | ``` 19 | 20 | Remove address info to make graphs easier to read 21 | 22 | ``` 23 | python cleanup_script.py out.perf > out2.perf 24 | ``` 25 | 26 | Visualize data with a flame graph 27 | ================================= 28 | 29 | ``` 30 | ./stackcollapse-perf.pl out2.perf > out.folded 31 | ./flamegraph.pl out.folded > test.svg 32 | ``` 33 | -------------------------------------------------------------------------------- /misc/pipeline.lua: -------------------------------------------------------------------------------- 1 | -- example script demonstrating HTTP pipelining 2 | 3 | init = function(args) 4 | local r = {} 5 | r[1] = wrk.format(nil, "/") 6 | r[2] = wrk.format(nil, "/") 7 | r[3] = wrk.format(nil, "/") 8 | r[4] = wrk.format(nil, "/") 9 | r[5] = wrk.format(nil, "/") 10 | r[6] = wrk.format(nil, "/") 11 | r[7] = wrk.format(nil, "/") 12 | r[8] = wrk.format(nil, "/") 13 | r[9] = wrk.format(nil, "/") 14 | r[10] = wrk.format(nil, "/") 15 | r[11] = wrk.format(nil, "/") 16 | r[12] = wrk.format(nil, "/") 17 | r[13] = wrk.format(nil, "/") 18 | r[14] = wrk.format(nil, "/") 19 | r[15] = wrk.format(nil, "/") 20 | r[16] = wrk.format(nil, "/") 21 | r[17] = wrk.format(nil, "/") 22 | r[18] = wrk.format(nil, "/") 23 | r[19] = wrk.format(nil, "/") 24 | r[20] = wrk.format(nil, "/") 25 | r[21] = wrk.format(nil, "/") 26 | r[22] = wrk.format(nil, "/") 27 | r[23] = wrk.format(nil, "/") 28 | r[24] = wrk.format(nil, "/") 29 | req = table.concat(r) 30 | end 31 | 32 | request = function() 33 | return req 34 | end 35 | -------------------------------------------------------------------------------- /misc/report.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import sys 3 | import os 4 | import json 5 | 6 | 7 | def report(samples, pid): 8 | plt.figure(figsize=(25, 10)) 9 | 10 | x = [s['t'] for s in samples if s['type'] == 'proc'] 11 | 12 | lines = [s for s in samples if s['type'] == 'event'] 13 | 14 | # minuss = min(s['uss'] for s in samples if s['type'] == 'proc') 15 | ussplot = plt.subplot(211) 16 | ussplot.set_title('uss') 17 | ussplot.plot( 18 | x, [s['uss'] for s in samples if s['type'] == 'proc'], '.') 19 | for l in lines: 20 | # ussplot.text(l['t'], minuss, l['event'], horizontalalignment='right', 21 | # rotation=-90, rotation_mode='anchor') 22 | ussplot.axvline(l['t']) 23 | 24 | connplot = plt.subplot(212) 25 | connplot.set_title('conn') 26 | connplot.plot( 27 | x, [s['conn'] for s in samples if s['type'] == 'proc'], '.') 28 | 29 | os.makedirs('.reports', exist_ok=True) 30 | path = '.reports/{}.png'.format(pid) 31 | plt.savefig(path) 32 | 33 | return path 34 | 35 | 36 | def load(filepath): 37 | samples = [] 38 | with open(filepath) as fp: 39 | for line in fp: 40 | line = line.strip() 41 | samples.append(json.loads(line)) 42 | 43 | return samples 44 | 45 | 46 | def order(samples): 47 | return sorted(samples, key=lambda x: x['t']) 48 | 49 | 50 | def normalize_time(samples): 51 | if not samples: 52 | return [] 53 | 54 | base_time = samples[0]['t'] 55 | 56 | return [{**s, 't': s['t'] - base_time} for s in samples] 57 | 58 | 59 | def main(): 60 | samples = load(sys.argv[1]) 61 | pid, _ = os.path.splitext(os.path.basename(sys.argv[1])) 62 | samples = order(samples) 63 | samples = normalize_time(samples) 64 | report(samples, pid) 65 | 66 | 67 | if __name__ == '__main__': 68 | main() 69 | -------------------------------------------------------------------------------- /misc/requirements-test.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | hypothesis==3.80.0 3 | psutil==5.6.6 4 | pytest==3.9.2 5 | pytoml==0.1.20 6 | perf==1.5.1 7 | cffi==1.11.5 8 | -------------------------------------------------------------------------------- /misc/requirements.txt: -------------------------------------------------------------------------------- 1 | uvloop>=0.11.3 2 | -------------------------------------------------------------------------------- /misc/rpm-requirements.txt: -------------------------------------------------------------------------------- 1 | libasan 2 | lcov 3 | libubsan 4 | -------------------------------------------------------------------------------- /misc/runpytest.py: -------------------------------------------------------------------------------- 1 | from pytest import main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /misc/simple.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import argparse 3 | import os.path 4 | import sys 5 | import socket 6 | 7 | sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/../../src')) 8 | 9 | import japronto.protocol.handler # noqa 10 | from japronto.router.cmatcher import Matcher # noqa 11 | from japronto.router import Router # noqa 12 | from japronto.app import Application # noqa 13 | 14 | 15 | def slash(request): 16 | return request.Response(text='Hello slash!') 17 | 18 | 19 | def hello(request): 20 | return request.Response(text='Hello hello!') 21 | 22 | 23 | async def sleep(request): 24 | await asyncio.sleep(3) 25 | return request.Response(text='I am sleepy') 26 | 27 | 28 | async def loop(request): 29 | i = 0 30 | while i < 10: 31 | await asyncio.sleep(1) 32 | print(i) 33 | i += 1 34 | 35 | return request.Response(text='Loop finished') 36 | 37 | 38 | def dump(request): 39 | sock = request.transport.get_extra_info('socket') 40 | no_delay = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) 41 | text = """ 42 | Method: {0.method} 43 | Path: {0.path} 44 | Version: {0.version} 45 | Headers: {0.headers} 46 | Match: {0.match_dict} 47 | Body: {0.body} 48 | QS: {0.query_string} 49 | query: {0.query} 50 | mime_type: {0.mime_type} 51 | encoding: {0.encoding} 52 | form: {0.form} 53 | keep_alive: {0.keep_alive} 54 | no_delay: {1} 55 | route: {0.route} 56 | """.strip().format(request, no_delay) 57 | 58 | return request.Response(text=text, headers={'X-Version': '123'}) 59 | 60 | 61 | app = Application() 62 | 63 | r = app.router 64 | r.add_route('/', slash) 65 | r.add_route('/hello', hello) 66 | r.add_route('/dump/{this}/{that}', dump) 67 | r.add_route('/sleep/{pinch}', sleep) 68 | r.add_route('/loop', loop) 69 | 70 | 71 | if __name__ == '__main__': 72 | argparser = argparse.ArgumentParser('server') 73 | argparser.add_argument( 74 | '-p', dest='flavor', default='block') 75 | args = argparser.parse_args(sys.argv[1:]) 76 | 77 | app.run(protocol_factory=japronto.protocol.handler.make_class(args.flavor)) 78 | -------------------------------------------------------------------------------- /misc/suppr.txt: -------------------------------------------------------------------------------- 1 | # This is a known leak. 2 | # Python leaks a little, we use malloc directly 3 | leak:PyMem_RawMalloc 4 | leak:_PyObject_GC_Resize 5 | leak:PyThread_allocate_lock 6 | leak:resize_compact 7 | -------------------------------------------------------------------------------- /misc/travis/before_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export JAPR_MSG=`git show -s --format=%B | xargs` 4 | export JAPR_WHEEL=`[[ $JAPR_MSG == *"[travis-wheel]"* ]] && echo 1 || echo 0` 5 | export JAPR_OS=`uname` 6 | 7 | if [[ $VERSION == "3.5."* ]]; then 8 | export PYTHON_TAG=cp35-cp35m 9 | elif [[ $VERSION == "3.6."* ]]; then 10 | export PYTHON_TAG=cp36-cp36m 11 | elif [[ $VERSION == "3.7."* ]]; then 12 | export PYTHON_TAG=cp37-cp37m 13 | elif [[ $VERSION == "3.8."* ]]; then 14 | export PYTHON_TAG=cp38-cp38m 15 | fi 16 | 17 | env | grep "^JAPR_" 18 | -------------------------------------------------------------------------------- /misc/travis/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | if [[ $JAPR_OS == "Darwin" ]]; then 6 | /usr/bin/clang --version 7 | source misc/terryfy/travis_tools.sh 8 | source misc/terryfy/library_installers.sh 9 | clean_builds 10 | get_python_environment macpython $VERSION venv 11 | fi 12 | 13 | if [[ $JAPR_WHEEL == "1" ]]; then 14 | pip install twine 15 | 16 | if [[ $JAPR_OS == "Linux" ]]; then 17 | docker info 18 | docker pull quay.io/pypa/manylinux1_x86_64 19 | fi 20 | fi 21 | -------------------------------------------------------------------------------- /misc/travis/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | if [[ $JAPR_WHEEL == "1" ]]; then 6 | if [[ $JAPR_OS == "Linux" ]]; then 7 | docker run --rm -u `id -u` -w /io -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /opt/python/$PYTHON_TAG/bin/python setup.py bdist_wheel 8 | docker run --rm -u `id -u` -w /io -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 auditwheel repair dist/*-$PYTHON_TAG-linux_x86_64.whl 9 | rm -r dist/* 10 | cp wheelhouse/*.whl dist 11 | fi 12 | 13 | if [[ $JAPR_OS == "Darwin" ]]; then 14 | python setup.py bdist_wheel 15 | fi 16 | 17 | ls -lha dist 18 | unzip -l dist/*.whl 19 | twine upload -u squeaky dist/*.whl 20 | fi 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Japronto 3 | """ 4 | import codecs 5 | import os 6 | import re 7 | 8 | from setuptools import setup, find_packages 9 | 10 | import build 11 | 12 | 13 | with codecs.open(os.path.join(os.path.abspath(os.path.dirname( 14 | __file__)), 'src', 'japronto', '__init__.py'), 'r', 'latin1') as fp: 15 | try: 16 | version = re.findall(r"^__version__ = '([^']+)'\r?$", 17 | fp.read(), re.M)[0] 18 | except IndexError: 19 | raise RuntimeError('Unable to determine version.') 20 | 21 | 22 | setup( 23 | name='japronto', 24 | version=version, 25 | url='http://github.com/squeaky-pl/japronto/', 26 | license='MIT', 27 | author='Paweł Piotr Przeradowski', 28 | author_email='przeradowski@gmail.com', 29 | description='A HTTP application toolkit and server bundle ' + 30 | 'based on uvloop and picohttpparser', 31 | package_dir={'': 'src'}, 32 | packages=find_packages('src'), 33 | keywords=['web', 'asyncio'], 34 | platforms='x86_64 Linux and MacOS X', 35 | install_requires=[ 36 | 'uvloop>=0.11.3', 37 | ], 38 | entry_points=""" 39 | [console_scripts] 40 | japronto = japronto.__main__:main 41 | """, 42 | classifiers=[ 43 | 'Development Status :: 2 - Pre-Alpha', 44 | 'Intended Audience :: Developers', 45 | 'Environment :: Web Environment', 46 | 'License :: OSI Approved :: MIT License', 47 | 'Operating System :: MacOS :: MacOS X', 48 | 'Operating System :: POSIX :: Linux', 49 | 'Programming Language :: C', 50 | 'Programming Language :: Python :: 3.5', 51 | 'Programming Language :: Python :: 3.6', 52 | 'Programming Language :: Python :: 3.7', 53 | 'Programming Language :: Python :: 3.8', 54 | 'Programming Language :: Python :: Implementation :: CPython', 55 | 'Topic :: Internet :: WWW/HTTP' 56 | ], 57 | zip_safe=False, 58 | include_package_data=True, 59 | package_data={'picohttpparser': ['*.so']}, 60 | ext_modules=build.get_platform(), 61 | cmdclass={'build_ext': build.custom_build_ext} 62 | ) 63 | -------------------------------------------------------------------------------- /src/japronto/__init__.py: -------------------------------------------------------------------------------- 1 | from .app import Application # noqa 2 | from .router import RouteNotFoundException # noqa 3 | 4 | 5 | __version__ = '0.1.2' 6 | -------------------------------------------------------------------------------- /src/japronto/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | from .runner import get_parser, verify, run 5 | 6 | 7 | def main(): 8 | parser = get_parser() 9 | args = parser.parse_args() 10 | 11 | if not args.script: 12 | os.putenv('_JAPR_IGNORE_RUN', '1') 13 | 14 | if args.reload: 15 | os.execv( 16 | sys.executable, 17 | [sys.executable, '-m', 'japronto.reloader', *sys.argv[1:]]) 18 | 19 | if not args.script: 20 | os.environ['_JAPR_IGNORE_RUN'] = '1' 21 | 22 | attribute = verify(args) 23 | if not attribute: 24 | return 1 25 | 26 | run(attribute, args) 27 | 28 | 29 | sys.exit(main()) 30 | -------------------------------------------------------------------------------- /src/japronto/capsule.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | void* get_ptr_from_mod(const char* module_name, const char* attr_name, 5 | const char* capsule_name) 6 | { 7 | void* ptr; 8 | PyObject* module = NULL; 9 | PyObject* capsule = NULL; 10 | 11 | module = PyImport_ImportModule(module_name); 12 | if(!module) 13 | goto error; 14 | 15 | capsule = PyObject_GetAttrString(module, attr_name); 16 | if(!capsule) 17 | goto error; 18 | 19 | ptr = PyCapsule_GetPointer(capsule, capsule_name); 20 | if(!ptr) 21 | goto error; 22 | 23 | goto finally; 24 | 25 | error: 26 | ptr = NULL; 27 | 28 | finally: 29 | Py_XDECREF(capsule); 30 | Py_XDECREF(module); 31 | return ptr; 32 | } 33 | 34 | 35 | PyObject* put_ptr_in_mod(PyObject* m, void* ptr, const char* attr_name, 36 | const char* capsule_name) 37 | { 38 | PyObject* capsule = NULL; 39 | 40 | capsule = PyCapsule_New(ptr, capsule_name, NULL); 41 | if(!capsule) 42 | goto error; 43 | 44 | if(PyModule_AddObject(m, attr_name, capsule) == -1) 45 | goto error; 46 | 47 | Py_INCREF(capsule); 48 | goto finally; 49 | 50 | error: 51 | Py_XDECREF(capsule); 52 | capsule = NULL; 53 | 54 | finally: 55 | return capsule; 56 | } 57 | -------------------------------------------------------------------------------- /src/japronto/capsule.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | 4 | void* get_ptr_from_mod(const char* module_name, const char* attr_name, 5 | const char* capsule_name); 6 | 7 | PyObject* put_ptr_in_mod(PyObject* m, void* ptr, const char* attr_name, 8 | const char* capsule_name); 9 | 10 | #define import_capi(module_name) \ 11 | get_ptr_from_mod(module_name, "_capi", module_name "._capi") 12 | 13 | #define export_capi(m, module_name, capi) \ 14 | put_ptr_in_mod(m, capi, "_capi", module_name "._capi") 15 | -------------------------------------------------------------------------------- /src/japronto/common.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | 4 | typedef enum { 5 | KEEP_ALIVE_UNSET, 6 | KEEP_ALIVE_TRUE, 7 | KEEP_ALIVE_FALSE 8 | } KEEP_ALIVE; 9 | -------------------------------------------------------------------------------- /src/japronto/cpu_features.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "cpu_features.h" 5 | 6 | int supports_x86_sse42(void) 7 | { 8 | #if defined(__clang__) 9 | unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; 10 | __get_cpuid(1, &eax, &ebx, &ecx, &edx); 11 | return ecx & bit_SSE42; 12 | #else 13 | __builtin_cpu_init(); 14 | return __builtin_cpu_supports("sse4.2"); 15 | #endif 16 | } 17 | -------------------------------------------------------------------------------- /src/japronto/cpu_features.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | int supports_x86_sse42(void); 4 | -------------------------------------------------------------------------------- /src/japronto/parser/.gitignore: -------------------------------------------------------------------------------- 1 | libpicohttpparser.c 2 | -------------------------------------------------------------------------------- /src/japronto/parser/__init__.py: -------------------------------------------------------------------------------- 1 | header_errors = [ 2 | 'malformed_headers', 'incomplete_headers', 'invalid_headers', 3 | 'excessive_data'] 4 | body_errors = ['malformed_body', 'incomplete_body'] 5 | -------------------------------------------------------------------------------- /src/japronto/parser/build_libpicohttpparser.py: -------------------------------------------------------------------------------- 1 | import distutils.log 2 | distutils.log.set_verbosity(distutils.log.DEBUG) 3 | 4 | import os.path 5 | 6 | import cffi 7 | ffibuilder = cffi.FFI() 8 | 9 | shared_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 10 | '../../picohttpparser')) 11 | 12 | print(shared_path) 13 | 14 | ffibuilder.set_source("libpicohttpparser", """ 15 | #include "picohttpparser.h" 16 | """, libraries=['picohttpparser'], include_dirs=[shared_path], 17 | library_dirs=[shared_path], 18 | extra_link_args=['-Wl,-rpath=' + shared_path]) 19 | 20 | # extra_objects=[os.path.join(shared_path, 'picohttpparser.o')], 21 | # or a list of libraries to link with 22 | # (more arguments like setup.py's Extension class: 23 | # include_dirs=[..], extra_objects=[..], and so on) 24 | 25 | ffibuilder.cdef(""" 26 | struct phr_header { 27 | const char *name; 28 | size_t name_len; 29 | const char *value; 30 | size_t value_len; 31 | }; 32 | 33 | struct phr_chunked_decoder { 34 | size_t bytes_left_in_chunk; /* number of bytes left in current chunk */ 35 | char consume_trailer; /* if trailing headers should be consumed */ 36 | char _hex_count; 37 | char _state; 38 | }; 39 | 40 | int phr_parse_request(const char *buf, size_t len, const char **method, 41 | size_t *method_len, const char **path, 42 | size_t *path_len, int *minor_version, 43 | struct phr_header *headers, size_t *num_headers, 44 | size_t last_len); 45 | 46 | ssize_t phr_decode_chunked(struct phr_chunked_decoder *decoder, char *buf, 47 | size_t *bufsz); 48 | """) 49 | 50 | 51 | if __name__ == "__main__": 52 | ffibuilder.compile(verbose=True) 53 | -------------------------------------------------------------------------------- /src/japronto/parser/cffiparser.py: -------------------------------------------------------------------------------- 1 | from parser.libpicohttpparser import ffi, lib 2 | 3 | 4 | class HttpRequestParser(object): 5 | def __init__(self, on_headers, on_body, on_error): 6 | self.on_headers = on_headers 7 | self.on_body = on_body 8 | self.on_error = on_error 9 | 10 | self.c_method = ffi.new('char **') 11 | self.method_len = ffi.new('size_t *') 12 | self.c_path = ffi.new('char **') 13 | self.path_len = ffi.new('size_t *') 14 | self.minor_version = ffi.new('int *') 15 | self.c_headers = ffi.new('struct phr_header[10]') 16 | self.num_headers = ffi.new('size_t *') 17 | self.chunked_offset = ffi.new('size_t*') 18 | 19 | self._reset_state(True) 20 | 21 | def _reset_state(self, disconnect=False): 22 | self.state = 'headers' 23 | self.transfer = None 24 | self.content_length = None 25 | self.chunked_decoder = None 26 | self.chunked_offset[0] = 0 27 | if disconnect: 28 | self.connection = None 29 | self.buffer = bytearray() 30 | 31 | def _parse_headers(self): 32 | if self.connection == 'close': 33 | self.on_error('excessive_data') 34 | self._reset_state(True) 35 | 36 | return -1 37 | 38 | self.num_headers[0] = 10 39 | 40 | # FIXME: More than 10 headers 41 | 42 | result = lib.phr_parse_request( 43 | ffi.from_buffer(self.buffer), len(self.buffer), 44 | self.c_method, self.method_len, 45 | self.c_path, self.path_len, 46 | self.minor_version, self.c_headers, self.num_headers, 0) 47 | 48 | if result == -2: 49 | return result 50 | elif result == -1: 51 | self.on_error('malformed_headers') 52 | self._reset_state(True) 53 | 54 | return result 55 | else: 56 | self._reset_state() 57 | 58 | method = ffi.cast( 59 | 'char[{}]'.format(self.method_len[0]), self.c_method[0]) 60 | path = ffi.cast( 61 | 'char[{}]'.format(self.path_len[0]), self.c_path[0]) 62 | headers = ffi.cast( 63 | "struct phr_header[{}]".format(self.num_headers[0]), 64 | self.c_headers) 65 | 66 | if ffi.buffer(method)[:] in (b'GET', b'DELETE', b'HEAD'): 67 | self.no_semantics = True 68 | 69 | if self.minor_version[0] == 0: 70 | self.connection = 'close' 71 | else: 72 | self.connection = 'keep-alive' 73 | 74 | for header in headers: 75 | header_name = ffi.string(header.name, header.name_len).title() 76 | # maybe len + strcasecmp C style is faster? 77 | if header_name == b'Transfer-Encoding': 78 | self.transfer = ffi.string( 79 | header.value, header.value_len).decode('ascii') 80 | # FIXME comma separated and invalid values 81 | elif header_name == b'Connection': 82 | self.connection = ffi.string( 83 | header.value, header.value_len).decode('ascii') 84 | # FIXME other options for Connection like updgrade 85 | elif header_name == b'Content-Length': 86 | content_length_error = False 87 | 88 | if not header.value_len: 89 | content_length_error = True 90 | 91 | if not content_length_error: 92 | content_length = ffi.buffer(header.value, header.value_len) 93 | 94 | if not content_length_error and content_length[0] in b'+-': 95 | content_length_error = True 96 | 97 | if not content_length_error: 98 | try: 99 | self.content_length = int(content_length[:]) 100 | except ValueError: 101 | content_length_error = True 102 | 103 | if content_length_error: 104 | self.on_error('invalid_headers') 105 | self._reset_state(True) 106 | 107 | return -1 108 | 109 | self.on_headers(method, path, self.minor_version[0], headers) 110 | 111 | self.buffer = self.buffer[result:] 112 | 113 | return result 114 | 115 | def _parse_body(self): 116 | if self.content_length is None and self.transfer is None: 117 | self.on_body(None) 118 | return 0 119 | elif self.content_length == 0: 120 | self.on_body(ffi.from_buffer(b"")) 121 | return 0 122 | elif self.content_length is not None: 123 | if self.content_length > len(self.buffer): 124 | return -2 125 | 126 | body = memoryview(self.buffer)[:self.content_length] 127 | self.on_body(ffi.from_buffer(body)) 128 | self.buffer = self.buffer[self.content_length:] 129 | 130 | result = self.content_length 131 | 132 | return result 133 | elif self.transfer == 'chunked': 134 | if not self.chunked_decoder: 135 | self.chunked_decoder = ffi.new('struct phr_chunked_decoder*') 136 | self.chunked_decoder.consume_trailer = b'\x01' 137 | 138 | chunked_offset_start = self.chunked_offset[0] 139 | self.chunked_offset[0] = len(self.buffer) - self.chunked_offset[0] 140 | result = lib.phr_decode_chunked( 141 | self.chunked_decoder, 142 | ffi.from_buffer(self.buffer) + chunked_offset_start, 143 | self.chunked_offset) 144 | self.chunked_offset[0] = self.chunked_offset[0] \ 145 | + chunked_offset_start 146 | 147 | if result == -2: 148 | self.buffer = self.buffer[:self.chunked_offset[0]] 149 | return result 150 | elif result == -1: 151 | self.on_error('malformed_body') 152 | self._reset_state(True) 153 | 154 | return result 155 | 156 | body = memoryview(self.buffer)[:self.chunked_offset[0]] 157 | self.on_body(ffi.from_buffer(body)) 158 | self.buffer = self.buffer[ 159 | self.chunked_offset[0]:self.chunked_offset[0] + result] 160 | self._reset_state() 161 | 162 | return result 163 | 164 | def feed(self, data): 165 | self.buffer += data 166 | 167 | while self.buffer: 168 | if self.state == 'headers': 169 | result = self._parse_headers() 170 | 171 | if result <= 0: 172 | return None 173 | 174 | self.state = 'body' 175 | 176 | if self.state == 'body': 177 | result = self._parse_body() 178 | 179 | if result < 0: 180 | return None 181 | 182 | self.state = 'headers' 183 | 184 | def feed_disconnect(self): 185 | if self.state == 'headers' and self.buffer: 186 | self.on_error('incomplete_headers') 187 | elif self.state == 'body': 188 | self.on_error('incomplete_body') 189 | 190 | self._reset_state(True) 191 | -------------------------------------------------------------------------------- /src/japronto/parser/cparser.gcda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/src/japronto/parser/cparser.gcda -------------------------------------------------------------------------------- /src/japronto/parser/cparser.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "picohttpparser.h" 7 | 8 | 9 | enum Parser_state { 10 | PARSER_HEADERS, 11 | PARSER_BODY 12 | }; 13 | 14 | 15 | enum Parser_transfer { 16 | PARSER_TRANSFER_UNSET, 17 | PARSER_IDENTITY, 18 | PARSER_CHUNKED 19 | }; 20 | 21 | 22 | enum Parser_connection { 23 | PARSER_CONNECTION_UNSET, 24 | PARSER_CLOSE, 25 | PARSER_KEEP_ALIVE 26 | }; 27 | 28 | #define PARSER_INITIAL_BUFFER_SIZE 4096 29 | 30 | typedef struct { 31 | #ifdef PARSER_STANDALONE 32 | PyObject_HEAD 33 | #endif 34 | 35 | enum Parser_state state; 36 | enum Parser_transfer transfer; 37 | enum Parser_connection connection; 38 | 39 | unsigned long content_length; 40 | struct phr_chunked_decoder chunked_decoder; 41 | size_t chunked_offset; 42 | 43 | char* buffer; 44 | size_t buffer_start; 45 | size_t buffer_end; 46 | size_t buffer_capacity; 47 | char inline_buffer[PARSER_INITIAL_BUFFER_SIZE]; 48 | 49 | #ifdef PARSER_STANDALONE 50 | PyObject* on_headers; 51 | PyObject* on_body; 52 | PyObject* on_error; 53 | #else 54 | void* protocol; 55 | #endif 56 | } Parser; 57 | 58 | #ifndef PARSER_STANDALONE 59 | void 60 | Parser_new(Parser* self); 61 | 62 | int 63 | Parser_init(Parser* self, void* protocol); 64 | 65 | void 66 | Parser_dealloc(Parser* self); 67 | 68 | Parser* 69 | Parser_feed(Parser* self, PyObject* py_data); 70 | 71 | Parser* 72 | Parser_feed_disconnect(Parser* self); 73 | 74 | int 75 | cparser_init(void); 76 | #endif 77 | -------------------------------------------------------------------------------- /src/japronto/parser/cparser_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | return Extension( 6 | 'japronto.parser.cparser', 7 | sources=['cparser.c', '../cpu_features.c'], 8 | include_dirs=['../../picohttpparser', '..'], 9 | extra_objects=[ 10 | 'src/picohttpparser/picohttpparser.o', 11 | 'src/picohttpparser/ssepicohttpparser.o'], 12 | define_macros=[('PARSER_STANDALONE', 1)]) 13 | -------------------------------------------------------------------------------- /src/japronto/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | class Pipeline: 2 | def __init__(self, ready): 3 | self._queue = [] 4 | self._ready = ready 5 | 6 | @property 7 | def empty(self): 8 | return not self._queue 9 | 10 | def queue(self, task): 11 | print("queued") 12 | 13 | self._queue.append(task) 14 | 15 | task.add_done_callback(self._task_done) 16 | 17 | def _task_done(self, task): 18 | print('Done', task.result()) 19 | 20 | pop_idx = 0 21 | for task in self._queue: 22 | if not task.done(): 23 | break 24 | 25 | self.write(task) 26 | 27 | pop_idx += 1 28 | 29 | if pop_idx: 30 | self._queue[:pop_idx] = [] 31 | 32 | def write(self, task): 33 | self._ready(task) 34 | print('Written', task.result()) 35 | 36 | 37 | if __name__ == '__main__': 38 | import asyncio 39 | 40 | async def coro(sleep): 41 | await asyncio.sleep(sleep) 42 | 43 | return sleep 44 | 45 | from uvloop import new_event_loop 46 | 47 | loop = new_event_loop() 48 | asyncio.set_event_loop(loop) 49 | 50 | pipeline = Pipeline() 51 | 52 | def queue(x): 53 | t = loop.create_task(coro(x)) 54 | pipeline.queue(t) 55 | 56 | loop.call_later(2, lambda: queue(2)) 57 | loop.call_later(12, lambda: queue(2)) 58 | 59 | queue(1) 60 | queue(10) 61 | queue(5) 62 | queue(1) 63 | 64 | loop.run_forever() 65 | -------------------------------------------------------------------------------- /src/japronto/pipeline/cpipeline.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | 7 | #ifdef PIPELINE_PAIR 8 | 9 | typedef struct { 10 | bool is_task; 11 | PyObject* request; 12 | PyObject* task; 13 | } PipelineEntry; 14 | 15 | static inline bool 16 | PipelineEntry_is_task(PipelineEntry entry) 17 | { 18 | return entry.is_task; 19 | } 20 | 21 | static inline void 22 | PipelineEntry_DECREF(PipelineEntry entry) 23 | { 24 | Py_DECREF(entry.request); 25 | // if not real task this was response, 26 | // that was inside request that was already freed above 27 | if(entry.is_task) 28 | Py_XDECREF(entry.task); 29 | } 30 | 31 | static inline void 32 | PipelineEntry_INCREF(PipelineEntry entry) 33 | { 34 | Py_INCREF(entry.request); 35 | Py_XINCREF(entry.task); 36 | } 37 | 38 | static inline PyObject* 39 | PipelineEntry_get_task(PipelineEntry entry) 40 | { 41 | return entry.task; 42 | } 43 | #else 44 | typedef PyObject* PipelineEntry; 45 | 46 | static inline bool 47 | PipelineEntry_is_task(PipelineEntry entry) 48 | { 49 | return true; 50 | } 51 | 52 | static inline void 53 | PipelineEntry_DECREF(PipelineEntry entry) 54 | { 55 | Py_DECREF(entry); 56 | } 57 | 58 | static inline void 59 | PipelineEntry_INCREF(PipelineEntry entry) 60 | { 61 | Py_INCREF(entry); 62 | } 63 | 64 | static inline PyObject* 65 | PipelineEntry_get_task(PipelineEntry entry) 66 | { 67 | return entry; 68 | } 69 | #endif 70 | 71 | 72 | typedef struct { 73 | PyObject_HEAD 74 | #ifdef PIPELINE_OPAQUE 75 | PyObject* ready; 76 | #else 77 | void* (*ready)(PipelineEntry, PyObject*); 78 | PyObject* protocol; 79 | #endif 80 | PyObject* task_done; 81 | PipelineEntry queue[10]; 82 | size_t queue_start; 83 | size_t queue_end; 84 | } Pipeline; 85 | 86 | 87 | #define PIPELINE_EMPTY(p) ((p)->queue_start == (p)->queue_end) 88 | 89 | #ifndef PIPELINE_OPAQUE 90 | PyObject* 91 | Pipeline_new(Pipeline* self); 92 | 93 | void 94 | Pipeline_dealloc(Pipeline* self); 95 | 96 | int 97 | Pipeline_init(Pipeline* self, void* (*ready)(PipelineEntry, PyObject*), PyObject* protocol); 98 | 99 | PyObject* 100 | Pipeline_queue(Pipeline* self, PipelineEntry entry); 101 | 102 | void* 103 | Pipeline_cancel(Pipeline* self); 104 | 105 | void* 106 | cpipeline_init(void); 107 | #endif 108 | -------------------------------------------------------------------------------- /src/japronto/pipeline/cpipeline_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | return Extension( 6 | 'japronto.pipeline.cpipeline', 7 | sources=['cpipeline.c'], 8 | include_dirs=[], 9 | libraries=[], library_dirs=[], 10 | extra_link_args=[], 11 | define_macros=[('PIPELINE_OPAQUE', 1)]) 12 | -------------------------------------------------------------------------------- /src/japronto/pipeline/test_pipeline.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import gc 3 | import sys 4 | from collections import namedtuple 5 | from functools import partial 6 | 7 | import pytest 8 | import uvloop 9 | 10 | from japronto.pipeline import Pipeline 11 | from japronto.pipeline.cpipeline import Pipeline as CPipeline 12 | 13 | 14 | Example = namedtuple('Example', 'value,delay') 15 | 16 | 17 | class FakeLoop: 18 | def call_soon(self, callback, val): 19 | callback(val) 20 | 21 | def get_debug(self): 22 | return False 23 | 24 | def create_future(self): 25 | return asyncio.Future(loop=self) 26 | 27 | 28 | class FakeFuture: 29 | cnt = 0 30 | 31 | def __new__(cls): 32 | print('new') 33 | cls.cnt += 1 34 | return object.__new__(cls) 35 | 36 | def __del__(self): 37 | type(self).cnt -= 1 38 | print('del') 39 | 40 | def __init__(self): 41 | self.callbacks = [] 42 | 43 | def add_done_callback(self, cb): 44 | self.callbacks.append(cb) 45 | 46 | def done(self): 47 | return hasattr(self, '_result') 48 | 49 | def result(self): 50 | return self._result 51 | 52 | def set_result(self, result): 53 | self._result = result 54 | 55 | for cb in self.callbacks: 56 | cb(self) 57 | 58 | self.callbacks = [] 59 | 60 | 61 | def parametrize_make_pipeline(): 62 | def make_pipeline(cls): 63 | results = [] 64 | 65 | def append(task): 66 | results.append(task.result()) 67 | 68 | return cls(append), results 69 | 70 | return pytest.mark.parametrize( 71 | 'make_pipeline', 72 | [partial(make_pipeline, CPipeline), partial(make_pipeline, Pipeline)], 73 | ids=['c', 'py']) 74 | 75 | 76 | def parametrize_case(examples): 77 | cases = [parse_case(i) for i in examples] 78 | 79 | return pytest.mark.parametrize('case', cases, ids=examples) 80 | 81 | 82 | def parse_example(e, accum): 83 | value, delay = map(int, e.split('@')) if '@' in e else (int(e), 0) 84 | 85 | return Example(value, delay + accum) 86 | 87 | 88 | def parse_case(case): 89 | results = [] 90 | delay = 0 91 | for c in case.split('+'): 92 | e = parse_example(c, delay) 93 | results.append(e) 94 | delay = e.delay 95 | 96 | return results 97 | 98 | 99 | def create_futures(resolves, case): 100 | futures = [None] * len(case) 101 | case = case[:] 102 | for c in sorted(case): 103 | idx = case.index(c) 104 | futures[idx] = resolves[idx]() 105 | case[idx] = None 106 | 107 | return tuple(futures) 108 | 109 | 110 | @parametrize_case([ 111 | '1', 112 | '1+5', '5+1', 113 | '1+5+10', '10+5+1', '5+1+10', '10+1+5', 114 | '1+10+5+1', '1+1+10+5', '10+5+1+1', '1+1+5+10' 115 | ]) 116 | @parametrize_make_pipeline() 117 | def test_fake_future(make_pipeline, case): 118 | pipeline, results = make_pipeline() 119 | 120 | def queue(x): 121 | fut = FakeFuture() 122 | pipeline.queue(fut) 123 | 124 | def resolve(): 125 | fut.set_result(x) 126 | return fut 127 | 128 | return resolve 129 | 130 | resolves = tuple(queue(v) for v in case) 131 | futures = create_futures(resolves, case) 132 | 133 | assert pipeline.empty 134 | 135 | del resolves 136 | 137 | # this loop is not pythonic on purpose 138 | # carefully don't create extra references 139 | for i in range(len(futures)): 140 | print(sys.getrefcount(futures[i])) 141 | del i 142 | 143 | assert results == case 144 | 145 | gc.collect() 146 | 147 | del futures 148 | 149 | gc.set_debug(gc.DEBUG_LEAK) 150 | gc.collect() 151 | 152 | print(gc.garbage) 153 | gc.set_debug(0) 154 | 155 | assert FakeFuture.cnt == 0 156 | 157 | 158 | def parametrize_loop(): 159 | return pytest.mark.parametrize( 160 | 'loop', [uvloop.new_event_loop(), asyncio.new_event_loop()], 161 | ids=['uv', 'aio']) 162 | 163 | 164 | @parametrize_case([ 165 | '1', '1@1', 166 | '1+2', '2+1', '2+1@1', '1@1+2', 167 | '1+2+3', '3+2+1', '2+1+3', '3+1+2', 168 | '1+3+2+1', '1+3+2+1+1@1', '1+1+3+2', '3+2+1+1', '1+1+2+3' 169 | ]) 170 | @parametrize_make_pipeline() 171 | @parametrize_loop() 172 | def test_real_task(loop, make_pipeline, case): 173 | DIVISOR = 1000 174 | pipeline, results = make_pipeline() 175 | 176 | async def coro(example): 177 | await asyncio.sleep(example.value / DIVISOR, loop=loop) 178 | 179 | return example 180 | 181 | def queue(x): 182 | task = loop.create_task(coro(x)) 183 | pipeline.queue(task) 184 | 185 | for v in case: 186 | if v.delay: 187 | loop.call_later(v.delay / DIVISOR, partial(queue, v)) 188 | else: 189 | queue(v) 190 | 191 | duration = max((e.value + e.delay) / DIVISOR for e in case) 192 | loop.run_until_complete(asyncio.sleep(duration, loop=loop)) 193 | 194 | # timing issue, wait a little bit more so we collect all the results 195 | if len(results) < len(case): 196 | loop.run_until_complete(asyncio.sleep(10 / DIVISOR, loop=loop)) 197 | 198 | assert pipeline.empty 199 | assert results == case 200 | -------------------------------------------------------------------------------- /src/japronto/protocol/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/src/japronto/protocol/__init__.py -------------------------------------------------------------------------------- /src/japronto/protocol/cprotocol.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef PARSER_STANDALONE 4 | #include "cparser.h" 5 | #endif 6 | 7 | #include "cpipeline.h" 8 | #include "crequest.h" 9 | #include 10 | 11 | #define GATHER_MAX_RESP 24 12 | 13 | typedef struct { 14 | PyObject* responses[GATHER_MAX_RESP]; 15 | size_t responses_end; 16 | size_t len; 17 | PyBytesObject* prev_buffer; 18 | bool enabled; 19 | } Gather; 20 | 21 | 22 | typedef struct { 23 | PyObject_HEAD 24 | 25 | #ifdef PARSER_STANDALONE 26 | PyObject* feed; 27 | PyObject* feed_disconnect; 28 | #else 29 | Parser parser; 30 | #endif 31 | Request static_request; 32 | Pipeline pipeline; 33 | #ifdef REAPER_ENABLED 34 | unsigned long idle_time; 35 | unsigned long read_ops; 36 | unsigned long last_read_ops; 37 | #endif 38 | PyObject* app; 39 | PyObject* matcher; 40 | PyObject* error_handler; 41 | PyObject* transport; 42 | PyObject* write; 43 | PyObject* writelines; 44 | PyObject* create_task; 45 | PyObject* request_logger; 46 | #ifdef PROTOCOL_TRACK_REFCNT 47 | Py_ssize_t none_cnt; 48 | Py_ssize_t true_cnt; 49 | Py_ssize_t false_cnt; 50 | #endif 51 | bool closed; 52 | Gather gather; 53 | } Protocol; 54 | 55 | #define GATHER_MAX_LEN (4096 - sizeof(PyBytesObject)) 56 | 57 | #ifndef PARSER_STANDALONE 58 | Protocol* Protocol_on_incomplete(Protocol* self); 59 | Protocol* Protocol_on_headers(Protocol*, char* method, size_t method_len, 60 | char* path, size_t path_len, int minor_version, 61 | void* headers, size_t num_headers); 62 | Protocol* Protocol_on_body(Protocol*, char* body, size_t body_len, size_t tail_len); 63 | Protocol* Protocol_on_error(Protocol*, PyObject*); 64 | #endif 65 | 66 | typedef struct { 67 | void* (*Protocol_close) 68 | (Protocol* self); 69 | } Protocol_CAPI; 70 | -------------------------------------------------------------------------------- /src/japronto/protocol/cprotocol_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | cparser = system.get_extension_by_path('japronto/parser/cparser_ext.py') 6 | cpipeline = system.get_extension_by_path( 7 | 'japronto/pipeline/cpipeline_ext.py') 8 | 9 | define_macros = [('PIPELINE_PAIR', 1)] 10 | if system.args.enable_reaper: 11 | define_macros.append(('REAPER_ENABLED', 1)) 12 | 13 | return Extension( 14 | 'japronto.protocol.cprotocol', 15 | sources=[ 16 | 'cprotocol.c', '../capsule.c', '../request/crequest.c', 17 | '../response/cresponse.c', 18 | *cparser.sources, *cpipeline.sources], 19 | include_dirs=[ 20 | '.', '..', '../parser', '../pipeline', 21 | '../router', '../request', 22 | '../response', *cparser.include_dirs], 23 | extra_objects=cparser.extra_objects, 24 | define_macros=define_macros) 25 | -------------------------------------------------------------------------------- /src/japronto/protocol/creaper.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "cprotocol.h" 4 | #include "capsule.h" 5 | 6 | typedef struct { 7 | PyObject_HEAD 8 | 9 | PyObject* connections; 10 | PyObject* call_later; 11 | PyObject* check_idle; 12 | PyObject* check_idle_handle; 13 | PyObject* check_interval; 14 | unsigned long idle_timeout; 15 | } Reaper; 16 | 17 | #ifdef REAPER_DEBUG_PRINT 18 | #define debug_print(format, ...) printf("reaper: " format "\n", __VA_ARGS__) 19 | #else 20 | #define debug_print(format, ...) 21 | #endif 22 | 23 | static Protocol_CAPI* protocol_capi; 24 | 25 | const long DEFAULT_CHECK_INTERVAL = 10; 26 | const unsigned long DEFAULT_IDLE_TIMEOUT = 60; 27 | 28 | static PyObject* default_check_interval; 29 | 30 | static PyObject* 31 | Reaper_new(PyTypeObject* type, PyObject* args, PyObject* kwds) 32 | { 33 | Reaper* self = NULL; 34 | 35 | self = (Reaper*)type->tp_alloc(type, 0); 36 | if(!self) 37 | goto finally; 38 | 39 | self->connections = NULL; 40 | self->call_later = NULL; 41 | self->check_idle = NULL; 42 | self->check_idle_handle = NULL; 43 | self->check_interval = NULL; 44 | 45 | finally: 46 | return (PyObject*)self; 47 | } 48 | 49 | 50 | static void 51 | Reaper_dealloc(Reaper* self) 52 | { 53 | Py_XDECREF(self->check_interval); 54 | Py_XDECREF(self->check_idle_handle); 55 | Py_XDECREF(self->check_idle); 56 | Py_XDECREF(self->call_later); 57 | Py_XDECREF(self->connections); 58 | 59 | Py_TYPE(self)->tp_free((PyObject*)self); 60 | } 61 | 62 | #ifdef REAPER_ENABLED 63 | static inline void* 64 | Reaper_schedule_check_idle(Reaper* self) 65 | { 66 | Py_XDECREF(self->check_idle_handle); 67 | self->check_idle_handle = PyObject_CallFunctionObjArgs( 68 | self->call_later, self->check_interval, self->check_idle, NULL); 69 | 70 | return self->check_idle_handle; 71 | } 72 | #endif 73 | 74 | 75 | static PyObject* 76 | Reaper_stop(Reaper* self) 77 | { 78 | #ifdef REAPER_ENABLED 79 | void* result = Py_None; 80 | PyObject* cancel = NULL; 81 | 82 | if(!(cancel = PyObject_GetAttrString(self->check_idle_handle, "cancel"))) 83 | goto error; 84 | 85 | PyObject* tmp; 86 | if(!(tmp = PyObject_CallFunctionObjArgs(cancel, NULL))) 87 | goto error; 88 | Py_DECREF(tmp); 89 | 90 | goto finally; 91 | 92 | error: 93 | result = NULL; 94 | 95 | finally: 96 | Py_XINCREF(result); 97 | Py_XDECREF(cancel); 98 | return result; 99 | #else 100 | Py_RETURN_NONE; 101 | #endif 102 | } 103 | 104 | 105 | static int 106 | Reaper_init(Reaper* self, PyObject* args, PyObject* kwds) 107 | { 108 | PyObject* loop = NULL; 109 | int result = 0; 110 | 111 | PyObject* app = NULL; 112 | PyObject* idle_timeout = NULL; 113 | 114 | static char* kwlist[] = {"app", "check_interval", "idle_timeout", NULL}; 115 | 116 | if (!PyArg_ParseTupleAndKeywords( 117 | args, kwds, "|OOO", kwlist, &app, &self->check_interval, &idle_timeout)) 118 | goto error; 119 | 120 | assert(app); 121 | 122 | if(!self->check_interval) 123 | self->check_interval = default_check_interval; 124 | Py_INCREF(self->check_interval); 125 | 126 | assert(PyLong_AsLong(self->check_interval) >= 0); 127 | 128 | if(!idle_timeout) 129 | self->idle_timeout = DEFAULT_IDLE_TIMEOUT; 130 | else 131 | self->idle_timeout = PyLong_AsLong(idle_timeout); 132 | 133 | assert(self->idle_timeout >= 0); 134 | 135 | debug_print("check_interval %ld", PyLong_AsLong(self->check_interval)); 136 | debug_print("idle_timeout %ld", self->idle_timeout); 137 | 138 | if(!(loop = PyObject_GetAttrString(app, "_loop"))) 139 | goto error; 140 | 141 | if(!(self->call_later = PyObject_GetAttrString(loop, "call_later"))) 142 | goto error; 143 | 144 | if(!(self->connections = PyObject_GetAttrString(app, "_connections"))) 145 | goto error; 146 | 147 | #ifdef REAPER_ENABLED 148 | if(!(self->check_idle = PyObject_GetAttrString((PyObject*)self, "_check_idle"))) 149 | goto error; 150 | 151 | if(!Reaper_schedule_check_idle(self)) 152 | goto error; 153 | #endif 154 | 155 | goto finally; 156 | 157 | error: 158 | result = -1; 159 | 160 | finally: 161 | Py_XDECREF(loop); 162 | return result; 163 | } 164 | 165 | 166 | #ifdef REAPER_ENABLED 167 | static PyObject* 168 | Reaper__check_idle(Reaper* self, PyObject* args) 169 | { 170 | PyObject* result = Py_None; 171 | PyObject* iterator = NULL; 172 | Protocol* conn = NULL; 173 | 174 | if(!(iterator = PyObject_GetIter(self->connections))) 175 | goto error; 176 | 177 | unsigned long check_interval = PyLong_AsLong(self->check_interval); 178 | while((conn = (Protocol*)PyIter_Next(iterator))) { 179 | debug_print( 180 | "conn %p, idle_time %ld, read_ops %ld, last_read_ops %ld", 181 | conn, conn->idle_time, conn->read_ops, conn->last_read_ops); 182 | 183 | if(conn->read_ops == conn->last_read_ops) { 184 | conn->idle_time += check_interval; 185 | 186 | if(conn->idle_time >= self->idle_timeout) { 187 | if(!protocol_capi->Protocol_close(conn)) 188 | goto error; 189 | } 190 | } else { 191 | conn->idle_time = 0; 192 | conn->last_read_ops = conn->read_ops; 193 | } 194 | 195 | Py_DECREF(conn); 196 | } 197 | 198 | if(!Reaper_schedule_check_idle(self)) 199 | goto error; 200 | 201 | goto finally; 202 | 203 | error: 204 | result = NULL; 205 | 206 | finally: 207 | Py_XDECREF(conn); 208 | Py_XDECREF(iterator); 209 | Py_XINCREF(result); 210 | return result; 211 | } 212 | #endif 213 | 214 | 215 | static PyMethodDef Reaper_methods[] = { 216 | #ifdef REAPER_ENABLED 217 | {"_check_idle", (PyCFunction)Reaper__check_idle, METH_NOARGS, ""}, 218 | #endif 219 | {"stop", (PyCFunction)Reaper_stop, METH_NOARGS, ""}, 220 | {NULL} 221 | }; 222 | 223 | 224 | static PyTypeObject ReaperType = { 225 | PyVarObject_HEAD_INIT(NULL, 0) 226 | "creaper.Reaper", /* tp_name */ 227 | sizeof(Reaper), /* tp_basicsize */ 228 | 0, /* tp_itemsize */ 229 | (destructor)Reaper_dealloc, /* tp_dealloc */ 230 | 0, /* tp_print */ 231 | 0, /* tp_getattr */ 232 | 0, /* tp_setattr */ 233 | 0, /* tp_reserved */ 234 | 0, /* tp_repr */ 235 | 0, /* tp_as_number */ 236 | 0, /* tp_as_sequence */ 237 | 0, /* tp_as_mapping */ 238 | 0, /* tp_hash */ 239 | 0, /* tp_call */ 240 | 0, /* tp_str */ 241 | 0, /* tp_getattro */ 242 | 0, /* tp_setattro */ 243 | 0, /* tp_as_buffer */ 244 | Py_TPFLAGS_DEFAULT, /* tp_flags */ 245 | "Reaper", /* tp_doc */ 246 | 0, /* tp_traverse */ 247 | 0, /* tp_clear */ 248 | 0, /* tp_richcompare */ 249 | 0, /* tp_weaklistoffset */ 250 | 0, /* tp_iter */ 251 | 0, /* tp_iternext */ 252 | Reaper_methods, /* tp_methods */ 253 | 0, /* tp_members */ 254 | 0, /* tp_getset */ 255 | 0, /* tp_base */ 256 | 0, /* tp_dict */ 257 | 0, /* tp_descr_get */ 258 | 0, /* tp_descr_set */ 259 | 0, /* tp_dictoffset */ 260 | (initproc)Reaper_init, /* tp_init */ 261 | 0, /* tp_alloc */ 262 | Reaper_new, /* tp_new */ 263 | }; 264 | 265 | 266 | static PyModuleDef creaper = { 267 | PyModuleDef_HEAD_INIT, 268 | "creaper", 269 | "creaper", 270 | -1, 271 | NULL, NULL, NULL, NULL, NULL 272 | }; 273 | 274 | 275 | PyMODINIT_FUNC 276 | PyInit_creaper(void) 277 | { 278 | PyObject* m = NULL; 279 | default_check_interval = NULL; 280 | 281 | if (PyType_Ready(&ReaperType) < 0) 282 | goto error; 283 | 284 | m = PyModule_Create(&creaper); 285 | if(!m) 286 | goto error; 287 | 288 | Py_INCREF(&ReaperType); 289 | PyModule_AddObject(m, "Reaper", (PyObject*)&ReaperType); 290 | 291 | if(!(default_check_interval = PyLong_FromLong(DEFAULT_CHECK_INTERVAL))) 292 | goto error; 293 | 294 | protocol_capi = import_capi("japronto.protocol.cprotocol"); 295 | if(!protocol_capi) 296 | goto error; 297 | 298 | goto finally; 299 | 300 | error: 301 | Py_XDECREF(default_check_interval); 302 | m = NULL; 303 | 304 | finally: 305 | return m; 306 | } 307 | -------------------------------------------------------------------------------- /src/japronto/protocol/creaper_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | define_macros = [('PIPELINE_PAIR', 1)] 6 | if system.args.enable_reaper: 7 | define_macros.append(('REAPER_ENABLED', 1)) 8 | 9 | return Extension( 10 | 'japronto.protocol.creaper', 11 | sources=['creaper.c', '../capsule.c'], 12 | include_dirs=[ 13 | '../parser', '../../picohttpparser', 14 | '../pipeline', '../request', 15 | '../router', '../response', '..'], 16 | define_macros=define_macros) 17 | -------------------------------------------------------------------------------- /src/japronto/protocol/generator.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "generator.h" 4 | 5 | 6 | typedef struct _Generator { 7 | PyObject_HEAD 8 | 9 | PyObject* object; 10 | } Generator; 11 | 12 | 13 | static PyTypeObject GeneratorType; 14 | 15 | 16 | #ifdef GENERATOR_OPAQUE 17 | static PyObject* 18 | Generator_new(PyTypeObject* type, PyObject* args, PyObject* kw) 19 | #else 20 | PyObject* 21 | Generator_new(void) 22 | #endif 23 | { 24 | Generator* self = NULL; 25 | 26 | #ifdef GENERATOR_OPAQUE 27 | self = (Generator*)type->tp_alloc(type, 0); 28 | #else 29 | self = (Generator*)GeneratorType.tp_alloc(&GeneratorType, 0); 30 | #endif 31 | if(!self) 32 | goto finally; 33 | 34 | self->object = NULL; 35 | 36 | finally: 37 | return (PyObject*)self; 38 | } 39 | 40 | 41 | #ifdef GENERATOR_OPAQUE 42 | static void 43 | #else 44 | void 45 | #endif 46 | Generator_dealloc(Generator* self) 47 | { 48 | Py_XDECREF(self->object); 49 | 50 | Py_TYPE(self)->tp_free((PyObject*)self); 51 | } 52 | 53 | 54 | #ifdef GENERATOR_OPAQUE 55 | static int 56 | Generator_init(Generator* self, PyObject *args, PyObject *kw) 57 | #else 58 | int 59 | Generator_init(Generator* self, PyObject* object) 60 | #endif 61 | { 62 | int result = 0; 63 | 64 | #ifdef GENERATOR_OPAQUE 65 | if(!PyArg_ParseTuple(args, "O", &self->object)) 66 | goto error; 67 | #else 68 | self->object = object; 69 | #endif 70 | 71 | Py_INCREF(self->object); 72 | 73 | goto finally; 74 | 75 | #ifdef GENERATOR_OPAQUE 76 | error: 77 | result = -1; 78 | #endif 79 | finally: 80 | return result; 81 | } 82 | 83 | 84 | static PyObject* 85 | Generator_next(Generator* self) 86 | { 87 | PyErr_SetObject(PyExc_StopIteration, self->object); 88 | 89 | return NULL; 90 | } 91 | 92 | 93 | static PyObject* 94 | Generator_send(Generator* self, PyObject* arg) 95 | { 96 | return Generator_next(self); 97 | } 98 | 99 | 100 | static PyMethodDef Generator_methods[] = { 101 | {"send", (PyCFunction)Generator_send, METH_O, ""}, 102 | {NULL} 103 | }; 104 | 105 | 106 | static PyTypeObject GeneratorType = { 107 | PyVarObject_HEAD_INIT(NULL, 0) 108 | "protocol.Generator", /* tp_name */ 109 | sizeof(Generator), /* tp_basicsize */ 110 | 0, /* tp_itemsize */ 111 | (destructor)Generator_dealloc, /* tp_dealloc */ 112 | 0, /* tp_print */ 113 | 0, /* tp_getattr */ 114 | 0, /* tp_setattr */ 115 | 0, /* tp_reserved */ 116 | 0, /* tp_repr */ 117 | 0, /* tp_as_number */ 118 | 0, /* tp_as_sequence */ 119 | 0, /* tp_as_mapping */ 120 | 0, /* tp_hash */ 121 | 0, /* tp_call */ 122 | 0, /* tp_str */ 123 | 0, /* tp_getattro */ 124 | 0, /* tp_setattro */ 125 | 0, /* tp_as_buffer */ 126 | Py_TPFLAGS_DEFAULT, /* tp_flags */ 127 | "Generator", /* tp_doc */ 128 | 0, /* tp_traverse */ 129 | 0, /* tp_clear */ 130 | 0, /* tp_richcompare */ 131 | 0, /* tp_weaklistoffset */ 132 | PyObject_SelfIter, /* tp_iter */ 133 | (iternextfunc)Generator_next, /* tp_iternext */ 134 | Generator_methods, /* tp_methods */ 135 | #ifdef GENERATOR_OPAQUE 136 | 0, /* tp_members */ 137 | 0, /* tp_getset */ 138 | 0, /* tp_base */ 139 | 0, /* tp_dict */ 140 | 0, /* tp_descr_get */ 141 | 0, /* tp_descr_set */ 142 | 0, /* tp_dictoffset */ 143 | (initproc)Generator_init, /* tp_init */ 144 | 0, /* tp_alloc */ 145 | Generator_new, /* tp_new */ 146 | #endif 147 | }; 148 | 149 | 150 | #ifdef GENERATOR_OPAQUE 151 | static PyModuleDef generator = { 152 | PyModuleDef_HEAD_INIT, 153 | "generator", 154 | "generator", 155 | -1, 156 | NULL, NULL, NULL, NULL, NULL 157 | }; 158 | #endif 159 | 160 | 161 | #ifdef GENERATOR_OPAQUE 162 | PyMODINIT_FUNC 163 | PyInit_generator(void) 164 | #else 165 | void* 166 | generator_init(void) 167 | #endif 168 | { 169 | #ifdef GENERATOR_OPAQUE 170 | PyObject* m = NULL; 171 | #else 172 | void* m = &GeneratorType; 173 | #endif 174 | 175 | if(PyType_Ready(&GeneratorType) < 0) 176 | goto error; 177 | 178 | #ifdef GENERATOR_OPAQUE 179 | m = PyModule_Create(&generator); 180 | if(!m) 181 | goto error; 182 | 183 | Py_INCREF(&GeneratorType); 184 | PyModule_AddObject(m, "Generator", (PyObject*)&GeneratorType); 185 | #endif 186 | 187 | goto finally; 188 | 189 | error: 190 | m = NULL; 191 | 192 | finally: 193 | return m; 194 | } 195 | -------------------------------------------------------------------------------- /src/japronto/protocol/generator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef GENERATOR_OPAQUE 4 | struct _Generator; 5 | 6 | #define GENERATOR struct _Generator 7 | 8 | PyObject* 9 | Generator_new(void); 10 | 11 | void 12 | Generator_dealloc(struct _Generator* self); 13 | 14 | int 15 | Generator_init(struct _Generator* self, PyObject* object); 16 | 17 | void* 18 | generator_init(void); 19 | #endif 20 | -------------------------------------------------------------------------------- /src/japronto/protocol/generator_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | return Extension( 6 | 'protocol.generator', 7 | sources=['generator.c'], 8 | include_dirs=[], 9 | define_macros=[('GENERATOR_OPAQUE', 1)]) 10 | -------------------------------------------------------------------------------- /src/japronto/protocol/handler.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from asyncio.queues import Queue 3 | 4 | 5 | from japronto.response.cresponse import Response 6 | from japronto.protocol.cprotocol import Protocol as CProtocol 7 | 8 | static_response = b"""HTTP/1.1 200 OK\r 9 | Connection: keep-alive\r 10 | Content-Length: 12\r 11 | Content-Type: text/plain; encoding=utf-8\r 12 | \r 13 | Hello statc! 14 | """ 15 | 16 | 17 | def make_class(flavor): 18 | if flavor == 'c': 19 | return CProtocol 20 | 21 | from japronto.parser import cparser 22 | 23 | class HttpProtocol(asyncio.Protocol): 24 | def __init__(self, loop, handler): 25 | self.parser = cparser.HttpRequestParser( 26 | self.on_headers, self.on_body, self.on_error) 27 | self.loop = loop 28 | self.response = Response() 29 | 30 | if flavor == 'queue': 31 | def connection_made(self, transport): 32 | self.transport = transport 33 | self.queue = Queue(loop=self.loop) 34 | self.loop.create_task(handle_requests(self.queue, transport)) 35 | else: 36 | def connection_made(self, transport): 37 | self.transport = transport 38 | 39 | def connection_lost(self, exc): 40 | self.parser.feed_disconnect() 41 | 42 | def data_received(self, data): 43 | self.parser.feed(data) 44 | 45 | def on_headers(self, request): 46 | return 47 | 48 | if flavor == 'block': 49 | def on_body(self, request): 50 | handle_request_block(request, self.transport, self.response) 51 | elif flavor == 'dump': 52 | def on_body(self, request): 53 | handle_dump(request, self.transport, self.response) 54 | elif flavor == 'task': 55 | def on_body(self, request): 56 | self.loop.create_task(handle_request(request, self.transport)) 57 | elif flavor == 'queue': 58 | def on_body(self, request): 59 | self.queue.put_nowait(request) 60 | elif flavor == 'inline': 61 | def on_body(self, request): 62 | body = 'Hello inlin!' 63 | status_code = 200 64 | mime_type = 'text/plain' 65 | encoding = 'utf-8' 66 | text = [b'HTTP/1.1 '] 67 | text.extend([str(status_code).encode(), b' OK\r\n']) 68 | text.append(b'Connection: keep-alive\r\n') 69 | text.append(b'Content-Length: ') 70 | text.extend([str(len(body)).encode(), b'\r\n']) 71 | text.extend([ 72 | b'Content-Type: ', mime_type.encode(), 73 | b'; encoding=', encoding.encode(), b'\r\n\r\n']) 74 | text.append(body.encode()) 75 | 76 | self.transport.write(b''.join(text)) 77 | 78 | elif flavor == 'static': 79 | def on_body(self, request): 80 | self.transport.write(static_response) 81 | 82 | def on_error(self, error): 83 | print(error) 84 | 85 | return HttpProtocol 86 | 87 | 88 | async def handle_requests(queue, transport): 89 | while 1: 90 | await queue.get() 91 | 92 | response = Response(text='Hello queue!') 93 | 94 | transport.write(response.render()) 95 | 96 | 97 | async def handle_request(request, transport): 98 | response = Response(text='Hello ttask!') 99 | 100 | transport.write(response.render()) 101 | 102 | 103 | def handle_request_block(request, transport, response): 104 | response.__init__(404, text='Hello block') 105 | 106 | transport.write(response.render()) 107 | 108 | 109 | def handle_dump(request, transport, response): 110 | text = request.path 111 | response.__init__(text=text) 112 | 113 | transport.write(response.render()) 114 | -------------------------------------------------------------------------------- /src/japronto/protocol/null.py: -------------------------------------------------------------------------------- 1 | class NullProtocol: 2 | def on_headers(self, *args): 3 | pass 4 | 5 | def on_body(self, body): 6 | pass 7 | 8 | def on_error(self, error): 9 | pass 10 | -------------------------------------------------------------------------------- /src/japronto/protocol/tracing.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | from parser.libpicohttpparser import ffi 4 | from request import HttpRequest 5 | 6 | 7 | class TracingProtocol: 8 | def __init__(self, on_headers_adapter: callable, 9 | on_body_adapter: callable): 10 | self.requests = [] 11 | self.error = None 12 | 13 | self.on_headers_adapter = on_headers_adapter 14 | self.on_body_adapter = on_body_adapter 15 | 16 | self.on_headers_call_count = 0 17 | self.on_body_call_count = 0 18 | self.on_error_call_count = 0 19 | 20 | def on_headers(self, *args): 21 | self.request = self.on_headers_adapter(*args) 22 | 23 | self.requests.append(self.request) 24 | 25 | self.on_headers_call_count += 1 26 | 27 | def on_body(self, body): 28 | self.request.body = self.on_body_adapter(body) 29 | 30 | self.on_body_call_count += 1 31 | 32 | def on_error(self, error: str): 33 | self.error = error 34 | 35 | self.on_error_call_count += 1 36 | 37 | 38 | def _request_from_cprotocol(method: memoryview, path: memoryview, version: int, 39 | headers: memoryview): 40 | method = method.tobytes().decode('ascii') 41 | path = path.tobytes().decode('ascii') 42 | version = "1.0" if version == 0 else "1.1" 43 | headers_len = headers.nbytes // ffi.sizeof("struct phr_header") 44 | headers_cdata = ffi.from_buffer(headers) 45 | headers_cdata = ffi.cast( 46 | 'struct phr_header[{}]'.format(headers_len), headers_cdata) 47 | 48 | headers = _extract_headers(headers_cdata) 49 | 50 | return HttpRequest(method, path, version, headers) 51 | 52 | 53 | def _body_from_cprotocol(body: memoryview): 54 | return None if body is None else body.tobytes() 55 | 56 | 57 | def _request_from_cffiprotocol(method: "char[]", path: "char[]", version: int, 58 | headers: "struct phr_header[]"): 59 | method = ffi.buffer(method)[:].decode('ascii') 60 | path = ffi.buffer(path)[:].decode('ascii') 61 | version = "1.0" if version == 0 else "1.1" 62 | 63 | headers = _extract_headers(headers) 64 | 65 | return HttpRequest(method, path, version, headers) 66 | 67 | 68 | def _body_from_cffiprotocol(body: "char[]"): 69 | return None if body is None else ffi.buffer(body)[:] 70 | 71 | 72 | def _extract_headers(headers_cdata: "struct phr_header[]"): 73 | headers = {} 74 | for header in headers_cdata: 75 | name = ffi.string(header.name, header.name_len).decode('ascii').title() 76 | value = ffi.string(header.value, header.value_len).decode('latin1') 77 | headers[name] = value 78 | 79 | return headers 80 | 81 | 82 | CTracingProtocol = partial( 83 | TracingProtocol, on_headers_adapter=_request_from_cprotocol, 84 | on_body_adapter=_body_from_cprotocol) 85 | 86 | 87 | CffiTracingProtocol = partial( 88 | TracingProtocol, on_headers_adapter=_request_from_cffiprotocol, 89 | on_body_adapter=_body_from_cffiprotocol) 90 | -------------------------------------------------------------------------------- /src/japronto/reloader.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os.path 3 | import os 4 | import time 5 | import threading 6 | import signal 7 | 8 | 9 | def main(): 10 | import subprocess 11 | 12 | terminating = False 13 | 14 | def signal_received(sig, frame): 15 | nonlocal terminating 16 | 17 | if sig == signal.SIGHUP: 18 | child.send_signal(signal.SIGHUP) 19 | else: 20 | terminating = True 21 | child.terminate() 22 | 23 | signal.signal(signal.SIGINT, signal_received) 24 | signal.signal(signal.SIGTERM, signal_received) 25 | signal.signal(signal.SIGHUP, signal_received) 26 | 27 | os.putenv('_JAPR_RELOADER', str(os.getpid())) 28 | 29 | while not terminating: 30 | child = subprocess.Popen([ 31 | sys.executable, '-m', 'japronto', 32 | '--reloader-pid', str(os.getpid()), 33 | *(v for v in sys.argv[1:] if v != '--reload')]) 34 | child.wait() 35 | if child.returncode != 0: 36 | break 37 | 38 | 39 | def exec_reloader(*, host, port, worker_num): 40 | args = ['--host', host, '--port', str(port)] 41 | if worker_num: 42 | args.extend(['--worker-num', str(worker_num)]) 43 | os.execv( 44 | sys.executable, 45 | [sys.executable, '-m', 'japronto.reloader', 46 | *args, '--script', *sys.argv]) 47 | 48 | 49 | def change_detector(): 50 | previous_mtimes = {} 51 | 52 | while 1: 53 | changed = False 54 | current_mtimes = {} 55 | 56 | for name, module in list(sys.modules.items()): 57 | try: 58 | filename = module.__file__ 59 | except AttributeError: 60 | continue 61 | 62 | if not filename.endswith('.py'): 63 | continue 64 | 65 | mtime = os.path.getmtime(filename) 66 | previous_mtime = previous_mtimes.get(name) 67 | if previous_mtime and previous_mtime != mtime: 68 | changed = True 69 | 70 | current_mtimes[name] = mtime 71 | 72 | yield changed 73 | 74 | previous_mtimes = current_mtimes 75 | 76 | 77 | class ChangeDetector(threading.Thread): 78 | def __init__(self, loop): 79 | super().__init__(daemon=True) 80 | self.loop = loop 81 | 82 | def run(self): 83 | for changed in change_detector(): 84 | if changed: 85 | self.loop.call_soon_threadsafe(self.loop.stop) 86 | os.kill(os.getppid(), signal.SIGHUP) 87 | return 88 | time.sleep(.5) 89 | 90 | 91 | if __name__ == "__main__": 92 | main() 93 | -------------------------------------------------------------------------------- /src/japronto/request/__init__.py: -------------------------------------------------------------------------------- 1 | import urllib.parse 2 | from json import loads as json_loads 3 | import cgi 4 | import encodings.idna 5 | import collections 6 | from http.cookies import _unquote as unquote_cookie 7 | 8 | 9 | class HttpRequest(object): 10 | __slots__ = ('path', 'method', 'version', 'headers', 'body') 11 | 12 | def __init__(self, method, path, version, headers): 13 | self.path = path 14 | self.method = method 15 | self.version = version 16 | self.headers = headers 17 | self.body = None 18 | 19 | def dump_headers(self): 20 | print('path', self.path) 21 | print('method', self.method) 22 | print('version', self.version) 23 | for n, v in self.headers.items(): 24 | print(n, v) 25 | 26 | def __repr__(self): 27 | return '' \ 28 | .format(self, len(self.headers)) 29 | 30 | 31 | def memoize(func): 32 | def wrapper(request): 33 | ns = request.extra.setdefault('_japronto', {}) 34 | try: 35 | return ns[func.__name__] 36 | except KeyError: 37 | pass 38 | 39 | result = func(request) 40 | ns[func.__name__] = result 41 | 42 | return result 43 | 44 | return wrapper 45 | 46 | 47 | @memoize 48 | def text(request): 49 | if request.body is None: 50 | return None 51 | 52 | return request.body.decode(request.encoding or 'utf-8') 53 | 54 | 55 | @memoize 56 | def json(request): 57 | if request.body is None: 58 | return None 59 | 60 | return json_loads(request.text) 61 | 62 | 63 | @memoize 64 | def query(request): 65 | qs = request.query_string 66 | if not qs: 67 | return {} 68 | return dict(urllib.parse.parse_qsl(qs)) 69 | 70 | 71 | def remote_addr(request): 72 | return request.transport.get_extra_info('peername')[0] 73 | 74 | 75 | @memoize 76 | def parsed_content_type(request): 77 | content_type = request.headers.get('Content-Type') 78 | if not content_type: 79 | return None, {} 80 | 81 | return cgi.parse_header(content_type) 82 | 83 | 84 | def mime_type(request): 85 | return parsed_content_type(request)[0] 86 | 87 | 88 | def encoding(request): 89 | return parsed_content_type(request)[1].get('charset') 90 | 91 | 92 | @memoize 93 | def parsed_form_and_files(request): 94 | if request.mime_type == 'application/x-www-form-urlencoded': 95 | return dict(urllib.parse.parse_qsl(request.text)), None 96 | elif request.mime_type == 'multipart/form-data': 97 | boundary = parsed_content_type(request)[1]['boundary'].encode('utf-8') 98 | return parse_multipart_form(request.body, boundary) 99 | 100 | return None, None 101 | 102 | 103 | def form(request): 104 | return parsed_form_and_files(request)[0] 105 | 106 | 107 | def files(request): 108 | return parsed_form_and_files(request)[1] 109 | 110 | 111 | @memoize 112 | def hostname_and_port(request): 113 | host = request.headers.get('Host') 114 | if not host: 115 | return None, None 116 | 117 | hostname, *rest = host.split(':', 1) 118 | port = rest[0] if rest else None 119 | 120 | return encodings.idna.ToUnicode(hostname), int(port) 121 | 122 | 123 | def port(request): 124 | return hostname_and_port(request)[1] 125 | 126 | 127 | def hostname(request): 128 | return hostname_and_port(request)[0] 129 | 130 | 131 | def parse_cookie(cookie): 132 | """Parse a ``Cookie`` HTTP header into a dict of name/value pairs. 133 | This function attempts to mimic browser cookie parsing behavior; 134 | it specifically does not follow any of the cookie-related RFCs 135 | (because browsers don't either). 136 | The algorithm used is identical to that used by Django version 1.9.10. 137 | """ 138 | cookiedict = {} 139 | for chunk in cookie.split(str(';')): 140 | if str('=') in chunk: 141 | key, val = chunk.split(str('='), 1) 142 | else: 143 | # Assume an empty name per 144 | # https://bugzilla.mozilla.org/show_bug.cgi?id=169091 145 | key, val = str(''), chunk 146 | key, val = key.strip(), val.strip() 147 | if key or val: 148 | # unquote using Python's algorithm. 149 | cookiedict[key] = unquote_cookie(val) 150 | return cookiedict 151 | 152 | 153 | @memoize 154 | def cookies(request): 155 | if 'Cookie' not in request.headers: 156 | return {} 157 | 158 | try: 159 | cookies = parse_cookie(request.headers['Cookie']) 160 | except Exception: 161 | return {} 162 | 163 | return {k: urllib.parse.unquote(v) for k, v in cookies.items()} 164 | 165 | 166 | File = collections.namedtuple('File', ['type', 'body', 'name']) 167 | 168 | 169 | def parse_multipart_form(body, boundary): 170 | files = {} 171 | fields = {} 172 | 173 | form_parts = body.split(boundary) 174 | for form_part in form_parts[1:-1]: 175 | file_name = None 176 | file_type = None 177 | field_name = None 178 | line_index = 2 179 | line_end_index = 0 180 | while not line_end_index == -1: 181 | line_end_index = form_part.find(b'\r\n', line_index) 182 | form_line = form_part[line_index:line_end_index].decode('utf-8') 183 | line_index = line_end_index + 2 184 | 185 | if not form_line: 186 | break 187 | 188 | colon_index = form_line.index(':') 189 | form_header_field = form_line[0:colon_index] 190 | form_header_value, form_parameters = cgi.parse_header( 191 | form_line[colon_index + 2:]) 192 | 193 | if form_header_field == 'Content-Disposition': 194 | if 'filename' in form_parameters: 195 | file_name = form_parameters['filename'] 196 | field_name = form_parameters.get('name') 197 | elif form_header_field == 'Content-Type': 198 | file_type = form_header_value 199 | 200 | post_data = form_part[line_index:-4] 201 | if file_name or file_type: 202 | file = File(type=file_type, name=file_name, body=post_data) 203 | files[field_name] = file 204 | else: 205 | value = post_data.decode('utf-8') 206 | fields[field_name] = value 207 | 208 | return fields, files 209 | -------------------------------------------------------------------------------- /src/japronto/request/crequest.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "cmatcher.h" 7 | #include "cresponse.h" 8 | #include "common.h" 9 | 10 | #define REQUEST_INITIAL_BUFFER_LEN 1024 11 | 12 | typedef struct { 13 | PyObject_HEAD 14 | 15 | char* method; 16 | size_t method_len; 17 | char* path; 18 | bool path_decoded; 19 | size_t path_len; 20 | bool qs_decoded; 21 | size_t qs_len; 22 | int minor_version; 23 | struct phr_header* headers; 24 | size_t num_headers; 25 | MatchDictEntry* match_dict_entries; 26 | size_t match_dict_length; 27 | char* body; 28 | size_t body_length; 29 | char* buffer; 30 | size_t buffer_len; 31 | char inline_buffer[REQUEST_INITIAL_BUFFER_LEN]; 32 | KEEP_ALIVE keep_alive; 33 | bool simple; 34 | bool response_called; 35 | MatcherEntry* matcher_entry; 36 | PyObject* exception; 37 | 38 | PyObject* transport; 39 | PyObject* app; 40 | PyObject* py_method; 41 | PyObject* py_path; 42 | PyObject* py_qs; 43 | PyObject* py_headers; 44 | PyObject* py_match_dict; 45 | PyObject* py_body; 46 | PyObject* extra; 47 | PyObject* done_callbacks; 48 | Response response; 49 | } Request; 50 | 51 | #define REQUEST(r) \ 52 | ((Request*)r) 53 | 54 | #define REQUEST_METHOD(r) \ 55 | REQUEST(r)->buffer 56 | 57 | #define REQUEST_PATH(r) \ 58 | REQUEST(r)->path 59 | 60 | 61 | typedef struct { 62 | PyTypeObject* RequestType; 63 | 64 | PyObject* (*Request_clone) 65 | (Request* original); 66 | 67 | void (*Request_from_raw) 68 | (Request* self, char* method, size_t method_len, 69 | char* path, size_t path_len, 70 | int minor_version, 71 | struct phr_header* headers, size_t num_headers); 72 | 73 | char* (*Request_get_decoded_path) 74 | (Request* self, size_t* path_len); 75 | 76 | void (*Request_set_match_dict_entries) 77 | (Request* self, MatchDictEntry* entries, size_t length); 78 | 79 | void (*Request_set_body) 80 | (Request* self, char* body, size_t body_len); 81 | } Request_CAPI; 82 | 83 | 84 | #ifndef REQUEST_OPAQUE 85 | PyObject* 86 | Request_new(PyTypeObject* type, Request* self); 87 | 88 | void 89 | Request_dealloc(Request* self); 90 | 91 | int 92 | Request_init(Request* self); 93 | 94 | void* 95 | crequest_init(void); 96 | #endif 97 | -------------------------------------------------------------------------------- /src/japronto/request/crequest_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | return Extension( 6 | 'japronto.request.crequest', 7 | sources=['crequest.c', '../response/cresponse.c', 8 | '../router/match_dict.c', '../capsule.c'], 9 | include_dirs=['../../picohttpparser', '..', 10 | '../response', '../router'], 11 | define_macros=[('REQUEST_OPAQUE', 1)]) 12 | -------------------------------------------------------------------------------- /src/japronto/response/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squeaky-pl/japronto/e73b76ea6ee2e2cc888da569f9caf6d59d1d3d8c/src/japronto/response/__init__.py -------------------------------------------------------------------------------- /src/japronto/response/cresponse.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include "common.h" 6 | 7 | 8 | #define RESPONSE_INITIAL_BUFFER_LEN 1024 9 | 10 | typedef struct { 11 | PyObject_HEAD 12 | 13 | bool opaque; 14 | int minor_version; 15 | KEEP_ALIVE keep_alive; 16 | 17 | PyObject* code; 18 | PyObject* mime_type; 19 | PyObject* body; 20 | PyObject* encoding; 21 | PyObject* headers; 22 | PyObject* cookies; 23 | 24 | char* buffer; 25 | size_t buffer_len; 26 | char inline_buffer[RESPONSE_INITIAL_BUFFER_LEN]; 27 | } Response; 28 | 29 | 30 | typedef struct { 31 | PyTypeObject* ResponseType; 32 | PyObject* (*Response_render)(Response*, bool); 33 | int (*Response_init)(Response* self, PyObject *args, PyObject *kw); 34 | } Response_CAPI; 35 | 36 | #ifndef RESPONSE_OPAQUE 37 | PyObject* 38 | Response_new(PyTypeObject* type, Response* self); 39 | 40 | void 41 | Response_dealloc(Response* self); 42 | #endif 43 | -------------------------------------------------------------------------------- /src/japronto/response/cresponse_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | define_macros = [('RESPONSE_OPAQUE', 1)] 6 | if system.args.enable_response_cache: 7 | define_macros.append(('RESPONSE_CACHE', 1)) 8 | 9 | return Extension( 10 | 'japronto.response.cresponse', 11 | sources=['cresponse.c', '../capsule.c'], 12 | include_dirs=['..'], 13 | define_macros=define_macros) 14 | -------------------------------------------------------------------------------- /src/japronto/response/py.py: -------------------------------------------------------------------------------- 1 | _responses = None 2 | 3 | 4 | def factory(status_code=200, text='', mime_type='text/plain', 5 | encoding='utf-8'): 6 | global _responses 7 | if _responses is None: 8 | _responses = [] 9 | for _ in range(100): 10 | _responses.append(Response()) 11 | 12 | response = _responses.pop() 13 | 14 | response.status_code = status_code 15 | response.mime_type = mime_type 16 | response.text = text 17 | response.encoding = encoding 18 | 19 | return response 20 | 21 | 22 | def dispose(response): 23 | _responses.append(response) 24 | 25 | 26 | class Response: 27 | __slots__ = ('status_code', 'mime_type', 'text', 'encoding') 28 | 29 | def __init__(self, status_code=200, text='', mime_type='text/plain', 30 | encoding='utf-8'): 31 | self.status_code = status_code 32 | self.mime_type = mime_type 33 | self.text = text 34 | self.encoding = encoding 35 | 36 | def render(self): 37 | data = ['HTTP/1.1 ', str(self.status_code), ' OK\r\n'] 38 | data.append('Connection: keep-alive\r\n') 39 | body = self.text.encode(self.encoding) 40 | data.extend([ 41 | 'Content-Type: ', self.mime_type, 42 | '; encoding=', self.encoding, '\r\n']) 43 | data.extend(['Content-Length: ', str(len(body)), '\r\n\r\n']) 44 | 45 | return ''.join(data).encode(self.encoding) + body 46 | -------------------------------------------------------------------------------- /src/japronto/response/reasons.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | static const char* reasons_1xx[] = { 4 | "Continue", //100 5 | "Switching Protocols", //101 6 | }; 7 | 8 | static const char* reasons_2xx[] = { 9 | "OK", //200 10 | "Created", //201 11 | "Accepted", //202 12 | "Non-Authoritative Information", //203 13 | "No Content", // 204 14 | "Reset Content", //205 15 | "Partial Content", //206 16 | }; 17 | 18 | static const char* reasons_3xx[] = { 19 | "Multiple Choices", //300 20 | "Moved Permanently", //301 21 | "Found", //302 22 | "See Other", //303, 23 | "Not Modified", //304 24 | "Use Proxy", //305 25 | "Proxy Switch", //306 26 | "Temporary Redirect", //307 27 | }; 28 | 29 | static const char* reasons_4xx[] = { 30 | "Bad Request", //400 31 | "Unauthorized", //401 32 | "Payment Required", //402 33 | "Forbidden", //403 34 | "Not Found", //404 35 | "Method Not Allowed", //405 36 | "Not acceptable", //406 37 | "Proxy Authentication Required", //407 38 | "Request Timeout", //408 39 | "Conflict", //409 40 | "Gone", //410 41 | "Length Required", //411 42 | "Precondition Failed", //412 43 | "Request Entity Too Large", //413 44 | "Request-URI Too Long", //414 45 | "Unsupported Media Type", //415 46 | "Requested Range Not Satisfiable", //416 47 | "Expectation Failed", //417 48 | }; 49 | 50 | static const char* reasons_5xx[] = { 51 | "Internal Server Error", //500 52 | "Not Implemented", //501 53 | "Bad Gateway", //502 54 | "Service Unavailable", //503 55 | "Gateway Timeout", //504 56 | "HTTP Version Not Supported", //505 57 | }; 58 | 59 | typedef struct { 60 | const char** reasons; 61 | size_t maximum; 62 | } ReasonRange; 63 | 64 | static const ReasonRange reason_ranges[] = { 65 | {reasons_1xx, 1}, 66 | {reasons_2xx, 6}, 67 | {reasons_3xx, 7}, 68 | {reasons_4xx, 17}, 69 | {reasons_5xx, 5} 70 | }; 71 | -------------------------------------------------------------------------------- /src/japronto/router/__init__.py: -------------------------------------------------------------------------------- 1 | from .route import Route, RouteNotFoundException 2 | from .cmatcher import Matcher 3 | 4 | 5 | class Router: 6 | def __init__(self, matcher_factory=Matcher): 7 | self._routes = [] 8 | self.matcher_factory = matcher_factory 9 | 10 | def add_route(self, pattern, handler, method=None, methods=None): 11 | assert not(method and methods), "Cannot use method and methods" 12 | 13 | if method: 14 | methods = [method] 15 | 16 | if not methods: 17 | methods = [] 18 | 19 | methods = {m.upper() for m in methods} 20 | route = Route(pattern, handler, methods) 21 | 22 | self._routes.append(route) 23 | 24 | return route 25 | 26 | def get_matcher(self): 27 | return self.matcher_factory(self._routes) 28 | -------------------------------------------------------------------------------- /src/japronto/router/analyzer.py: -------------------------------------------------------------------------------- 1 | import dis 2 | import functools 3 | import types 4 | 5 | 6 | FLAG_COROUTINE = 128 7 | 8 | 9 | def is_simple(fun): 10 | """A heuristic to find out if a function is simple enough.""" 11 | seen_load_fast_0 = False 12 | seen_load_response = False 13 | seen_call_fun = False 14 | 15 | for instruction in dis.get_instructions(fun): 16 | if instruction.opname == 'LOAD_FAST' and instruction.arg == 0: 17 | seen_load_fast_0 = True 18 | continue 19 | 20 | if instruction.opname == 'LOAD_ATTR' \ 21 | and instruction.argval == 'Response': 22 | seen_load_response = True 23 | continue 24 | 25 | if instruction.opname.startswith('CALL_FUNCTION'): 26 | if seen_call_fun: 27 | return False 28 | 29 | seen_call_fun = True 30 | continue 31 | 32 | return seen_call_fun and seen_load_fast_0 and seen_load_response 33 | 34 | 35 | def is_pointless_coroutine(fun): 36 | for instruction in dis.get_instructions(fun): 37 | if instruction.opname in ('GET_AWAITABLE', 'YIELD_FROM'): 38 | return False 39 | 40 | return True 41 | 42 | 43 | def coroutine_to_func(f): 44 | # Based on http://stackoverflow.com/questions/13503079/ 45 | # how-to-create-a-copy-of-a-python-function 46 | oc = f.__code__ 47 | code = types.CodeType( 48 | oc.co_argcount, oc.co_kwonlyargcount, oc.co_nlocals, oc.co_stacksize, 49 | oc.co_flags & ~FLAG_COROUTINE, 50 | oc.co_code, oc.co_consts, oc.co_names, oc.co_varnames, oc.co_filename, 51 | oc.co_name, oc.co_firstlineno, oc.co_lnotab, oc.co_freevars, 52 | oc.co_cellvars) 53 | g = types.FunctionType( 54 | code, f.__globals__, name=f.__name__, argdefs=f.__defaults__, 55 | closure=f.__closure__) 56 | g = functools.update_wrapper(g, f) 57 | g.__kwdefaults__ = f.__kwdefaults__ 58 | 59 | return g 60 | -------------------------------------------------------------------------------- /src/japronto/router/cmatcher.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "match_dict.h" 7 | 8 | typedef struct { 9 | PyObject* route; 10 | PyObject* handler; 11 | bool coro_func; 12 | bool simple; 13 | size_t pattern_len; 14 | size_t methods_len; 15 | size_t placeholder_cnt; 16 | char buffer[]; 17 | } MatcherEntry; 18 | 19 | 20 | typedef struct _Matcher Matcher; 21 | 22 | 23 | typedef struct { 24 | MatcherEntry* (*Matcher_match_request) 25 | (Matcher* matcher, PyObject* request, 26 | MatchDictEntry** match_dict_entries, size_t* match_dict_length); 27 | } Matcher_CAPI; 28 | -------------------------------------------------------------------------------- /src/japronto/router/cmatcher_ext.py: -------------------------------------------------------------------------------- 1 | from distutils.core import Extension 2 | 3 | 4 | def get_extension(): 5 | return Extension( 6 | 'japronto.router.cmatcher', 7 | sources=['cmatcher.c', 'match_dict.c', '../capsule.c'], 8 | include_dirs=['.', '../request', '..', '../response']) 9 | -------------------------------------------------------------------------------- /src/japronto/router/match_dict.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "match_dict.h" 4 | 5 | 6 | PyObject* 7 | MatchDict_entries_to_dict(MatchDictEntry* entries, size_t length) 8 | { 9 | PyObject* match_dict = NULL; 10 | if(!(match_dict = PyDict_New())) 11 | goto error; 12 | 13 | for(MatchDictEntry* entry = entries; entry < entries + length; entry++) { 14 | PyObject* key = NULL; 15 | PyObject* value = NULL; 16 | 17 | if(!(key = PyUnicode_FromStringAndSize(entry->key, entry->key_length))) 18 | goto loop_error; 19 | 20 | if(!(value = PyUnicode_FromStringAndSize(entry->value, entry->value_length))) 21 | goto loop_error; 22 | 23 | if(PyDict_SetItem(match_dict, key, value) == -1) 24 | goto loop_error; 25 | 26 | goto loop_finally; 27 | 28 | loop_error: 29 | Py_XDECREF(match_dict); 30 | match_dict = NULL; 31 | 32 | loop_finally: 33 | Py_XDECREF(key); 34 | Py_XDECREF(value); 35 | if(!match_dict) 36 | goto error; 37 | } 38 | 39 | goto finally; 40 | 41 | error: 42 | Py_XDECREF(match_dict); 43 | match_dict = NULL; 44 | 45 | finally: 46 | return match_dict; 47 | } 48 | -------------------------------------------------------------------------------- /src/japronto/router/match_dict.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | typedef struct { 6 | char* key; 7 | size_t key_length; 8 | char* value; 9 | size_t value_length; 10 | } MatchDictEntry; 11 | 12 | 13 | PyObject* 14 | MatchDict_entries_to_dict(MatchDictEntry* entries, size_t length); 15 | -------------------------------------------------------------------------------- /src/japronto/router/matcher.py: -------------------------------------------------------------------------------- 1 | class Matcher: 2 | def __init__(self, routes): 3 | self._routes = routes 4 | 5 | def match_request(self, request): 6 | for route in self._routes: 7 | match_dict = {} 8 | rest = request.path 9 | 10 | value = True 11 | for typ, data in route.segments: 12 | if typ == 'exact': 13 | if not rest.startswith(data): 14 | break 15 | 16 | rest = rest[len(data):] 17 | elif typ == 'placeholder': 18 | value, slash, rest = rest.partition('/') 19 | if not value: 20 | break 21 | match_dict[data] = value 22 | rest = slash + rest 23 | else: 24 | assert 0, 'Unknown type' 25 | 26 | if rest: 27 | continue 28 | 29 | if not value: 30 | continue 31 | 32 | if len(match_dict) != route.placeholder_cnt: 33 | continue 34 | 35 | if route.methods and request.method not in route.methods: 36 | continue 37 | 38 | return route, match_dict 39 | -------------------------------------------------------------------------------- /src/japronto/router/route.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from enum import IntEnum 3 | from struct import Struct 4 | 5 | from . import analyzer 6 | 7 | 8 | class RouteNotFoundException(Exception): 9 | pass 10 | 11 | 12 | class Route: 13 | def __init__(self, pattern, handler, methods): 14 | self.pattern = pattern 15 | self.handler = handler 16 | self.methods = methods 17 | self.segments = parse(pattern) 18 | self.placeholder_cnt = \ 19 | sum(1 for s in self.segments if s[0] == 'placeholder') 20 | 21 | def __repr__(self): 22 | return ''.format( 23 | self.pattern, self.methods, hex(id(self))) 24 | 25 | def describe(self): 26 | return self.pattern + (' ' if self.methods else '') + \ 27 | ' '.join(self.methods) 28 | 29 | def __eq__(self, other): 30 | return self.pattern == other.pattern and self.methods == other.methods 31 | 32 | 33 | def parse(pattern): 34 | names = set() 35 | result = [] 36 | 37 | rest = pattern 38 | while rest: 39 | exact = '' 40 | while rest: 41 | chunk, _, rest = rest.partition('{') 42 | exact += chunk 43 | if rest and rest[0] == '{': 44 | exact += '{{' 45 | rest = rest[1:] 46 | else: 47 | break 48 | 49 | if exact: 50 | exact = exact.replace('{{', '{').replace('}}', '}') 51 | result.append(('exact', exact)) 52 | if not rest: 53 | break 54 | 55 | name, _, rest = rest.partition('}') 56 | if not _: 57 | raise ValueError('Unbalanced "{" in pattern') 58 | if rest and rest[0] != '/': 59 | raise ValueError( 60 | '"}" must be followed by "/" or appear at the end') 61 | if name in names: 62 | raise ValueError('Duplicate name "{}" in pattern'.format(name)) 63 | names.add(name) 64 | result.append(('placeholder', name)) 65 | 66 | return result 67 | 68 | 69 | class SegmentType(IntEnum): 70 | EXACT = 0 71 | PLACEHOLDER = 1 72 | 73 | 74 | """ 75 | typedef struct { 76 | PyObject* route; 77 | PyObject* handler; 78 | bool coro_func; 79 | bool simple; 80 | size_t pattern_len; 81 | size_t methods_len; 82 | size_t placeholder_cnt; 83 | char buffer[]; 84 | } MatcherEntry; 85 | """ 86 | MatcherEntry = Struct('PP??NNN') 87 | 88 | """ 89 | typedef enum { 90 | SEGMENT_EXACT, 91 | SEGMENT_PLACEHOLDER 92 | } SegmentType; 93 | 94 | 95 | typedef struct { 96 | size_t data_length; 97 | char data[]; 98 | } ExactSegment; 99 | 100 | 101 | typedef struct { 102 | size_t name_length; 103 | char name[]; 104 | } PlaceholderSegment; 105 | 106 | 107 | typedef struct { 108 | SegmentType type; 109 | 110 | union { 111 | ExactSegment exact; 112 | PlaceholderSegment placeholder; 113 | }; 114 | } Segment; 115 | """ 116 | ExactSegment = Struct('iN') 117 | PlaceholderSegment = Struct('iN') 118 | Segment = Struct('iN') 119 | 120 | 121 | def roundto8(v): 122 | return (v + 7) & ~7 123 | 124 | 125 | def padto8(data): 126 | """Pads data to the multiplies of 8 bytes. 127 | 128 | This makes x86_64 faster and prevents 129 | undefined behavior on other platforms""" 130 | length = len(data) 131 | return data + b'\xdb' * (roundto8(length) - length) 132 | 133 | 134 | retain_handlers = set() 135 | 136 | 137 | def compile(route): 138 | pattern_buf = b'' 139 | for segment in route.segments: 140 | typ = getattr(SegmentType, segment[0].upper()) 141 | pattern_buf += Segment.pack(typ, len(segment[1].encode('utf-8'))) \ 142 | + padto8(segment[1].encode('utf-8')) 143 | methods_buf = ' '.join(route.methods).encode('ascii') 144 | methods_len = len(methods_buf) 145 | if methods_buf: 146 | methods_buf += b' ' 147 | methods_len += 1 148 | methods_buf = padto8(methods_buf) 149 | 150 | handler = route.handler 151 | if asyncio.iscoroutinefunction(handler) \ 152 | and analyzer.is_pointless_coroutine(handler): 153 | handler = analyzer.coroutine_to_func(handler) 154 | # since we save id to handler in matcher entry and this is the only 155 | # reference before INCREF-ed in matcher we store it in set to prevent 156 | # destruction 157 | retain_handlers.add(handler) 158 | 159 | return MatcherEntry.pack( 160 | id(route), id(handler), 161 | asyncio.iscoroutinefunction(handler), 162 | analyzer.is_simple(handler), 163 | len(pattern_buf), methods_len, route.placeholder_cnt) \ 164 | + pattern_buf + methods_buf 165 | 166 | 167 | def compile_all(routes): 168 | return b''.join(compile(r) for r in routes) 169 | -------------------------------------------------------------------------------- /src/japronto/router/test_analyzer.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import pytest 4 | 5 | from . import analyzer 6 | 7 | 8 | simple_fixtures = OrderedDict([ 9 | ('empty', ('def a(): pass', False)), 10 | ('arg', ('def a(b): return b', False)), 11 | ('simple', ('def a(c): return c.Response()', True)), 12 | ('body', ('def a(c): return c.Response(body="abc")', True)), 13 | ('wrongattr', ('def a(d): return d.R()', False)), 14 | ('extracall', ( 15 | ''' 16 | def a(b): 17 | d() 18 | return b.Response() 19 | 20 | def d(): 21 | pass 22 | ''', False)), 23 | ('expressions', ( 24 | ''' 25 | def a(b): 26 | c = "Hey!" 27 | d = "Dude" 28 | return b.Response(json={c: d}) 29 | ''', True)) 30 | ]) 31 | 32 | 33 | @pytest.mark.parametrize( 34 | 'code,simple', simple_fixtures.values(), ids=list(simple_fixtures.keys())) 35 | def test_is_simple(code, simple): 36 | module = compile(code, '?', 'exec') 37 | fun_code = module.co_consts[0] 38 | 39 | assert analyzer.is_simple(fun_code) == simple 40 | 41 | 42 | pointless_fixtures = OrderedDict([ 43 | ('empty', ('async def a(): pass', True)), 44 | ('simple', ('async def a(): return 1', True)), 45 | ('yieldfrom', ('def a(b): yield from b', False)), 46 | ('await', ('async def a(b): await b', False)) 47 | ]) 48 | 49 | 50 | @pytest.mark.parametrize( 51 | 'code,pointless', pointless_fixtures.values(), 52 | ids=list(pointless_fixtures.keys())) 53 | def test_is_pointless(code, pointless): 54 | module = compile(code, '?', 'exec') 55 | fun_code = module.co_consts[0] 56 | 57 | assert analyzer.is_pointless_coroutine(fun_code) == pointless 58 | -------------------------------------------------------------------------------- /src/japronto/router/test_matcher.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | import pytest 4 | 5 | from . import Route 6 | from .matcher import Matcher 7 | from .cmatcher import Matcher as CMatcher 8 | 9 | 10 | class FakeRequest: 11 | def __init__(self, method, path): 12 | self.method = method 13 | self.path = path 14 | 15 | @classmethod 16 | def from_str(cls, value): 17 | return cls(*value.split()) 18 | 19 | 20 | class TracingRoute(Route): 21 | cnt = 0 22 | 23 | def __new__(cls, *args, **kw): 24 | print('new', args) 25 | cls.cnt += 1 26 | return Route.__new__(cls) 27 | 28 | def __init__(self, pattern, methods): 29 | super().__init__(pattern, lambda x: None, methods=methods) 30 | 31 | def __del__(self): 32 | type(self).cnt -= 1 33 | print('del') 34 | 35 | 36 | def route_from_str(value): 37 | pattern, *methods = value.split() 38 | if methods: 39 | methods = methods[0].split(',') 40 | 41 | return TracingRoute(pattern, methods=methods) 42 | 43 | 44 | def parametrize_make_matcher(): 45 | def make(cls): 46 | routes = [route_from_str(r) for r in [ 47 | '/', 48 | '/test GET', 49 | '/hi/{there} POST,DELETE', 50 | '/{oh}/{dear} PATCH', 51 | '/lets PATCH' 52 | ]] 53 | 54 | return cls(routes) 55 | 56 | make_matcher = partial(make, Matcher) 57 | make_cmatcher = partial(make, CMatcher) 58 | 59 | return pytest.mark.parametrize( 60 | 'make_matcher', [make_matcher, make_cmatcher], ids=['py', 'c']) 61 | 62 | 63 | def parametrize_request_route_and_dict(cases): 64 | return pytest.mark.parametrize( 65 | 'req,route,match_dict', 66 | ((FakeRequest.from_str(req), route_from_str(route), match_dict) 67 | for req, route, match_dict in cases), 68 | ids=[req + '-' + route for req, route, _ in cases]) 69 | 70 | 71 | @parametrize_request_route_and_dict([ 72 | ('GET /', '/', {}), 73 | ('POST /', '/', {}), 74 | ('GET /test', '/test GET', {}), 75 | ('DELETE /hi/jane', '/hi/{there} POST,DELETE', {'there': 'jane'}), 76 | ('PATCH /lets/go', '/{oh}/{dear} PATCH', {'oh': 'lets', 'dear': 'go'}), 77 | ('PATCH /lets', '/lets PATCH', {}) 78 | ]) 79 | @parametrize_make_matcher() 80 | def test_matcher(make_matcher, req, route, match_dict): 81 | cnt = TracingRoute.cnt 82 | 83 | matcher = make_matcher() 84 | assert matcher.match_request(req) == (route, match_dict) 85 | del matcher 86 | 87 | assert cnt == TracingRoute.cnt 88 | 89 | 90 | def parametrize_request(requests): 91 | return pytest.mark.parametrize( 92 | 'req', (FakeRequest.from_str(r) for r in requests), ids=requests) 93 | 94 | 95 | @parametrize_request([ 96 | 'POST /test', 97 | 'GET /test/', 98 | 'GET /hi/jane', 99 | 'POST /hi/jane/', 100 | 'POST /hi/', 101 | 'GET /abc', 102 | 'PATCH //dance' 103 | ]) 104 | @parametrize_make_matcher() 105 | def test_matcher_not_found(make_matcher, req): 106 | cnt = TracingRoute.cnt 107 | 108 | matcher = make_matcher() 109 | assert matcher.match_request(req) is None 110 | del matcher 111 | 112 | assert cnt == TracingRoute.cnt 113 | -------------------------------------------------------------------------------- /src/japronto/router/test_route.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from collections import namedtuple 3 | 4 | import pytest 5 | 6 | from .route import parse, MatcherEntry, Segment, SegmentType, Route, \ 7 | compile, roundto8 8 | 9 | 10 | @pytest.mark.parametrize('pattern,result', [ 11 | ('/', [('exact', '/')]), 12 | ('/{{a}}', [('exact', '/{a}')]), 13 | ('{a}', [('placeholder', 'a')]), 14 | ('a/{a}', [('exact', 'a/'), ('placeholder', 'a')]), 15 | ('{a}/a', [('placeholder', 'a'), ('exact', '/a')]), 16 | ('{a}/{{a}}', [('placeholder', 'a'), ('exact', '/{a}')]), 17 | ('{a}/{b}', [('placeholder', 'a'), ('exact', '/'), ('placeholder', 'b')]) 18 | ]) 19 | def test_parse(pattern, result): 20 | assert parse(pattern) == result 21 | 22 | 23 | @pytest.mark.parametrize('pattern,error', [ 24 | ('{a', 'Unbalanced'), 25 | ('{a}/{b', 'Unbalanced'), 26 | ('{a}a', 'followed by'), 27 | ('{a}/{a}', 'Duplicate') 28 | ]) 29 | def test_parse_error(pattern, error): 30 | with pytest.raises(ValueError) as info: 31 | parse(pattern) 32 | assert error in info.value.args[0] 33 | 34 | 35 | DecodedRoute = namedtuple( 36 | 'DecodedRoute', 37 | 'route_id,handler_id,coro_func,simple,placeholder_cnt,segments,methods') 38 | 39 | 40 | def decompile(buffer): 41 | route_id, handler_id, coro_func, simple, \ 42 | pattern_len, methods_len, placeholder_cnt \ 43 | = MatcherEntry.unpack_from(buffer, 0) 44 | offset = MatcherEntry.size 45 | pattern_offset_end = offset + roundto8(pattern_len) 46 | 47 | segments = [] 48 | while offset < pattern_offset_end: 49 | typ, segment_length = Segment.unpack_from(buffer, offset) 50 | offset += Segment.size 51 | typ = SegmentType(typ).name.lower() 52 | data = buffer[offset:offset + segment_length].decode('utf-8') 53 | offset += roundto8(segment_length) 54 | 55 | segments.append((typ, data)) 56 | 57 | methods = buffer[offset:offset + methods_len].strip().decode('ascii') \ 58 | .split() 59 | 60 | return DecodedRoute( 61 | route_id, handler_id, coro_func, simple, 62 | placeholder_cnt, segments, methods) 63 | 64 | 65 | def handler(): 66 | pass 67 | 68 | 69 | async def coro(): 70 | # needs to have await to prevent being promoted to function 71 | await asyncio.sleep(1) 72 | 73 | 74 | @pytest.mark.parametrize('route', [ 75 | Route('/', handler, []), 76 | Route('/', coro, ['GET']), 77 | Route('/test/{hi}', handler, []), 78 | Route('/test/{hi}', coro, ['POST']), 79 | Route('/tést', coro, ['POST']) 80 | ], ids=Route.describe) 81 | def test_compile(route): 82 | decompiled = decompile(compile(route)) 83 | 84 | assert decompiled.route_id == id(route) 85 | assert decompiled.handler_id == id(route.handler) 86 | assert decompiled.coro_func == asyncio.iscoroutinefunction(route.handler) 87 | assert not decompiled.simple 88 | assert decompiled.placeholder_cnt == route.placeholder_cnt 89 | assert decompiled.segments == route.segments 90 | assert decompiled.methods == route.methods 91 | -------------------------------------------------------------------------------- /src/japronto/runner.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser, SUPPRESS 2 | from importlib import import_module 3 | import os 4 | import sys 5 | import runpy 6 | 7 | from .app import Application 8 | 9 | try: 10 | ModuleNotFoundError 11 | except NameError: 12 | ModuleNotFoundError = ImportError 13 | 14 | 15 | def get_parser(): 16 | prog = 'python -m japronto' if sys.argv[0].endswith('__main__.py') \ 17 | else 'japronto' 18 | parser = ArgumentParser(prog=prog) 19 | parser.add_argument('--host', dest='host', type=str, default='0.0.0.0') 20 | parser.add_argument('--port', dest='port', type=int, default=8080) 21 | parser.add_argument('--worker-num', dest='worker_num', type=int, default=1) 22 | parser.add_argument( 23 | '--reload', dest='reload', action='store_const', 24 | const=True, default=False) 25 | 26 | parser.add_argument( 27 | '--reloader-pid', dest='reloader_pid', type=int, help=SUPPRESS) 28 | parser.add_argument( 29 | '--script', dest='script', action='store_const', 30 | const=True, default=False, help=SUPPRESS) 31 | 32 | parser.add_argument('application') 33 | 34 | return parser 35 | 36 | 37 | def verify(args): 38 | if args.script: 39 | script = args.application 40 | 41 | if not os.path.exists(script): 42 | print("Script '{}' not found.".format(script)) 43 | 44 | return script 45 | else: 46 | try: 47 | module, attribute = args.application.rsplit('.', 1) 48 | except ValueError: 49 | print( 50 | "Application specificer must contain at least one '.'," + 51 | "got '{}'.".format(args.application)) 52 | return False 53 | 54 | try: 55 | module = import_module(module) 56 | except ModuleNotFoundError as e: 57 | print(e.args[0] + ' on Python search path.') 58 | return False 59 | 60 | try: 61 | attribute = getattr(module, attribute) 62 | except AttributeError: 63 | print( 64 | "Module '{}' does not have an attribute '{}'." 65 | .format(module.__name__, attribute)) 66 | return False 67 | 68 | if not isinstance(attribute, Application): 69 | print("{} is not an instance of 'japronto.Application'.") 70 | return False 71 | 72 | return attribute 73 | 74 | 75 | def run(attribute, args): 76 | if args.script: 77 | runpy.run_path(attribute) 78 | else: 79 | attribute._run( 80 | host=args.host, port=args.port, 81 | worker_num=args.worker_num, reloader_pid=args.reloader_pid) 82 | -------------------------------------------------------------------------------- /src/picohttpparser/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | gcc -c picohttpparser.c -O3 -fPIC -msse4.2 -o ssepicohttpparser.o 6 | gcc -c picohttpparser.c -O3 -fPIC -o picohttpparser.o 7 | gcc -fPIC -shared -o libpicohttpparser.so picohttpparser.o ssepicohttpparser.o 8 | -------------------------------------------------------------------------------- /src/picohttpparser/picohttpparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase, 3 | * Shigeo Mitsunari 4 | * 5 | * The software is licensed under either the MIT License (below) or the Perl 6 | * license. 7 | * 8 | * Permission is hereby granted, free of charge, to any person obtaining a copy 9 | * of this software and associated documentation files (the "Software"), to 10 | * deal in the Software without restriction, including without limitation the 11 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 12 | * sell copies of the Software, and to permit persons to whom the Software is 13 | * furnished to do so, subject to the following conditions: 14 | * 15 | * The above copyright notice and this permission notice shall be included in 16 | * all copies or substantial portions of the Software. 17 | * 18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 24 | * IN THE SOFTWARE. 25 | */ 26 | 27 | #ifndef picohttpparser_h 28 | #define picohttpparser_h 29 | 30 | #include 31 | 32 | #ifdef _MSC_VER 33 | #define ssize_t intptr_t 34 | #endif 35 | 36 | /* $Id: 67fd3ee74103ada60258d8a16e868f483abcca87 $ */ 37 | 38 | #ifdef __cplusplus 39 | extern "C" { 40 | #endif 41 | 42 | /* contains name and value of a header (name == NULL if is a continuing line 43 | * of a multiline header */ 44 | struct phr_header { 45 | const char *name; 46 | size_t name_len; 47 | const char *value; 48 | size_t value_len; 49 | }; 50 | 51 | /* returns number of bytes consumed if successful, -2 if request is partial, 52 | * -1 if failed */ 53 | int phr_parse_request(const char *buf, size_t len, const char **method, size_t *method_len, const char **path, size_t *path_len, 54 | int *minor_version, struct phr_header *headers, size_t *num_headers, size_t last_len); 55 | 56 | int phr_parse_request_sse42(const char *buf, size_t len, const char **method, size_t *method_len, const char **path, size_t *path_len, 57 | int *minor_version, struct phr_header *headers, size_t *num_headers, size_t last_len); 58 | 59 | 60 | /* ditto */ 61 | // squeaky_p: we don't use it yet 62 | #if 0 63 | int phr_parse_response(const char *_buf, size_t len, int *minor_version, int *status, const char **msg, size_t *msg_len, 64 | struct phr_header *headers, size_t *num_headers, size_t last_len); 65 | 66 | /* ditto */ 67 | int phr_parse_headers(const char *buf, size_t len, struct phr_header *headers, size_t *num_headers, size_t last_len); 68 | #endif 69 | 70 | /* should be zero-filled before start */ 71 | struct phr_chunked_decoder { 72 | size_t bytes_left_in_chunk; /* number of bytes left in current chunk */ 73 | char consume_trailer; /* if trailing headers should be consumed */ 74 | char _hex_count; 75 | char _state; 76 | }; 77 | 78 | /* the function rewrites the buffer given as (buf, bufsz) removing the chunked- 79 | * encoding headers. When the function returns without an error, bufsz is 80 | * updated to the length of the decoded data available. Applications should 81 | * repeatedly call the function while it returns -2 (incomplete) every time 82 | * supplying newly arrived data. If the end of the chunked-encoded data is 83 | * found, the function returns a non-negative number indicating the number of 84 | * octets left undecoded at the tail of the supplied buffer. Returns -1 on 85 | * error. 86 | */ 87 | ssize_t phr_decode_chunked(struct phr_chunked_decoder *decoder, char *buf, size_t *bufsz); 88 | 89 | /* returns if the chunked decoder is in middle of chunked data */ 90 | int phr_decode_chunked_is_in_data(struct phr_chunked_decoder *decoder); 91 | 92 | #ifdef __cplusplus 93 | } 94 | #endif 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /tutorial/1_hello.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | Make sure you have both [pip](https://pip.pypa.io/en/stable/installing/) and at 4 | least version 3.5 of Python before starting. On Linux or MacOS X you can install 5 | directly using pip. If you are on Windows you can still develop and use 6 | Japronto with [Docker](https://docs.docker.com/engine/installation/#/on-macos-and-windows). 7 | 8 | Installing 9 | ---------- 10 | 11 | On Linux and OSX install Japronto with `python3 -m pip install japronto`. 12 | On Windows or if you simply prefer Docker pull Japronto image with `docker pull japronto/japronto`. 13 | 14 | Creating your Hello world app 15 | ----------------------------- 16 | 17 | Copy and paste following code into a file named `hello.py`: 18 | 19 | ```python 20 | # examples/1_hello/hello.py 21 | from japronto import Application 22 | 23 | 24 | # Views handle logic, take request as a parameter and 25 | # returns Response object back to the client 26 | def hello(request): 27 | return request.Response(text='Hello world!') 28 | 29 | 30 | # The Application instance is a fundamental concept. 31 | # It is a parent to all the resources and all the settings 32 | # can be tweaked here. 33 | app = Application() 34 | 35 | # The Router instance lets you register your handlers and execute 36 | # them depending on the url path and methods 37 | app.router.add_route('/', hello) 38 | 39 | # Finally start our server and handle requests until termination is 40 | # requested. Enabling debug lets you see request logs and stack traces. 41 | app.run(debug=True) 42 | ``` 43 | 44 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 45 | 46 | Run it 47 | ------ 48 | 49 | On Linux and OSX run the server with just: `python3 hello.py`. 50 | 51 | If using Docker run `docker run -p 8080:8080 -v $(pwd)/hello.py:/hello.py japronto/japronto --script /hello.py`. This will mount local `hello.py` into container as `/hello.py` which is later passed to Docker entry point. 52 | 53 | Now open the address `http://localhost:8080` in your web browser. You should see the message *Hello world!*. 54 | 55 | You now have a working Japronto server! 56 | 57 | 58 | **Next:** [Asynchronous handlers](2_async.md) 59 | -------------------------------------------------------------------------------- /tutorial/2_async.md: -------------------------------------------------------------------------------- 1 | # Asynchronous handlers 2 | 3 | With Japronto you can freely combine synchronous and asynchronous handlers and 4 | fully take advantage of both ecosystems. Choose wisely when to use asynchronous 5 | programming. Unless you are connecting to third party APIs, want to run input-output tasks in the background, expect large 6 | latency or do long-running blocking queries to your database you are probably 7 | better off programming synchronously. 8 | 9 | 10 | ```python 11 | # examples/2_async/async.py 12 | import asyncio 13 | from japronto import Application 14 | 15 | 16 | # This is a synchronous handler. 17 | def synchronous(request): 18 | return request.Response(text='I am synchronous!') 19 | 20 | 21 | # This is an asynchronous handler, it spends most of the time in the event loop. 22 | # It wakes up every second 1 to print and finally returns after 3 seconds. 23 | # This does let other handlers to be executed in the same processes while 24 | # from the point of view of the client it took 3 seconds to complete. 25 | async def asynchronous(request): 26 | for i in range(1, 4): 27 | await asyncio.sleep(1) 28 | print(i, 'seconds elapsed') 29 | 30 | return request.Response(text='3 seconds elapsed') 31 | 32 | 33 | app = Application() 34 | 35 | r = app.router 36 | r.add_route('/sync', synchronous) 37 | r.add_route('/async', asynchronous) 38 | 39 | app.run() 40 | ``` 41 | 42 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 43 | 44 | 45 | **Next:** [Router](3_router.md) 46 | -------------------------------------------------------------------------------- /tutorial/3_router.md: -------------------------------------------------------------------------------- 1 | # Router 2 | 3 | The router is a subsystem responsible for directing incoming requests to 4 | particular handlers based on some conditions, namely the URL path 5 | and HTTP method. It's available under `router` property of an `Application` 6 | instance and presents `add_router` method which takes `path` pattern, `handler` 7 | and optionally one or more `method`s. 8 | 9 | 10 | ```python 11 | # examples/3_router/router.py 12 | from japronto import Application 13 | 14 | 15 | app = Application() 16 | r = app.router 17 | 18 | 19 | # Requests with the path set exactly to `/` and whatever method 20 | # will be directed here. 21 | def slash(request): 22 | return request.Response(text='Hello {} /!'.format(request.method)) 23 | 24 | 25 | r.add_route('/', slash) 26 | 27 | 28 | # Requests with the path set exactly to '/love' and the method 29 | # set exactly to `GET` will be directed here. 30 | def get_love(request): 31 | return request.Response(text='Got some love') 32 | 33 | 34 | r.add_route('/love', get_love, 'GET') 35 | 36 | 37 | # Requests with the path set exactly to '/methods' and the method 38 | # set to `POST` or `DELETE` will be directed here. 39 | def methods(request): 40 | return request.Response(text=request.method) 41 | 42 | 43 | r.add_route('/methods', methods, methods=['POST', 'DELETE']) 44 | 45 | 46 | # Requests with the path starting with `/params/` segment and followed 47 | # by two additional segments will be directed here. 48 | # Values of the additional segments will be stored in side `request.match_dict` 49 | # dictionary with keys taken from {} placeholders. A request to `/params/1/2` 50 | # would leave `match_dict` set to `{'p1': 1, 'p2': '2'}`. 51 | def params(request): 52 | return request.Response(text=str(request.match_dict)) 53 | 54 | 55 | r.add_route('/params/{p1}/{p2}', params) 56 | 57 | app.run() 58 | ``` 59 | 60 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 61 | 62 | **Next:** [Request object](4_request.md) 63 | -------------------------------------------------------------------------------- /tutorial/4_request.md: -------------------------------------------------------------------------------- 1 | # Request Object 2 | 3 | Request represents an incoming HTTP request with a rich set of properties. They can be divided into 4 | three categories: Request line and headers, message body and miscellaneous. 5 | 6 | ```python 7 | # examples/4_request/request.py 8 | from json import JSONDecodeError 9 | 10 | from japronto import Application 11 | 12 | 13 | # Request line and headers. 14 | # This represents the part of a request that comes before message body. 15 | # Given a HTTP 1.1 `GET` request to `/basic?a=1` this would yield 16 | # `method` set to `GET`, `path` set to `/basic`, `version` set to `1.1` 17 | # `query_string` set to `a=1` and `query` set to `{'a': '1'}`. 18 | # Additionally if headers are sent they will be present in `request.headers` 19 | # dictionary. The keys are normalized to standard `Camel-Cased` convention. 20 | def basic(request): 21 | text = """Basic request properties: 22 | Method: {0.method} 23 | Path: {0.path} 24 | HTTP version: {0.version} 25 | Query string: {0.query_string} 26 | Query: {0.query}""".format(request) 27 | 28 | if request.headers: 29 | text += "\nHeaders:\n" 30 | for name, value in request.headers.items(): 31 | text += " {0}: {1}\n".format(name, value) 32 | 33 | return request.Response(text=text) 34 | 35 | 36 | # Message body 37 | # If there is a message body attached to a request (as in a case of `POST`) 38 | # method the following attributes can be used to examine it. 39 | # Given a `POST` request with body set to `b'J\xc3\xa1'`, `Content-Length` header set 40 | # to `3` and `Content-Type` header set to `text/plain; charset=utf-8` this 41 | # would yield `mime_type` set to `'text/plain'`, `encoding` set to `'utf-8'`, 42 | # `body` set to `b'J\xc3\xa1'` and `text` set to `'Já'`. 43 | # `form` and `files` attributes are dictionaries respectively used for HTML forms and 44 | # HTML file uploads. The `json` helper property will try to decode `body` as a 45 | # JSON document and give you resulting Python data type. 46 | def body(request): 47 | text = """Body related properties: 48 | Mime type: {0.mime_type} 49 | Encoding: {0.encoding} 50 | Body: {0.body} 51 | Text: {0.text} 52 | Form parameters: {0.form} 53 | Files: {0.files} 54 | """.format(request) 55 | 56 | try: 57 | json = request.json 58 | except JSONDecodeError: 59 | pass 60 | else: 61 | text += "\nJSON:\n" 62 | text += str(json) 63 | 64 | return request.Response(text=text) 65 | 66 | 67 | # Miscellaneous 68 | # `route` will point to an instance of `Route` object representing 69 | # route chosen by router to handle this request. `hostname` and `port` 70 | # represent parsed `Host` header if any. `remote_addr` is the address of 71 | # a client or reverse proxy. If `keep_alive` is true the client requested to 72 | # keep connection open after the response is delivered. `match_dict` contains 73 | # route placeholder values as documented in `2_router.md`. `cookies` contains 74 | # a dictionary of HTTP cookies if any. 75 | def misc(request): 76 | text = """Miscellaneous: 77 | Matched route: {0.route} 78 | Hostname: {0.hostname} 79 | Port: {0.port} 80 | Remote address: {0.remote_addr}, 81 | HTTP Keep alive: {0.keep_alive} 82 | Match parameters: {0.match_dict} 83 | """.strip().format(request) 84 | 85 | if request.cookies: 86 | text += "\nCookies:\n" 87 | for name, value in request.cookies.items(): 88 | text += " {0}: {1}\n".format(name, value) 89 | 90 | return request.Response(text=text) 91 | 92 | 93 | app = Application() 94 | app.router.add_route('/basic', basic) 95 | app.router.add_route('/body', body) 96 | app.router.add_route('/misc', misc) 97 | app.run() 98 | ``` 99 | 100 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 101 | 102 | 103 | **Next:** [Response object](5_response.md) 104 | -------------------------------------------------------------------------------- /tutorial/5_response.md: -------------------------------------------------------------------------------- 1 | # Response object 2 | 3 | Handlers return Response instances to fulfill requests. They can contain status code, headers and almost always a body. 4 | At the moment Response instances are immutable once created, this 5 | restriction will be lifted in a next version. 6 | 7 | ```python 8 | # examples/5_response/response.py 9 | import random 10 | from http.cookies import SimpleCookie 11 | 12 | from japronto.app import Application 13 | 14 | 15 | # Providing just text argument yields a `text/plain` response 16 | # encoded with `utf8` codec (charset set accordingly) 17 | def text(request): 18 | return request.Response(text='Hello world!') 19 | 20 | 21 | # You can override encoding by providing `encoding` attribute. 22 | def encoding(request): 23 | return request.Response(text='Já pronto!', encoding='iso-8859-1') 24 | 25 | 26 | # You can also set a custom MIME type. 27 | def mime(request): 28 | return request.Response( 29 | mime_type="image/svg+xml", 30 | text=""" 31 | 32 | 33 | 34 | """) 35 | 36 | 37 | # Or serve binary data. `Content-Type` set to `application/octet-stream` 38 | # automatically but you can always provide your own `mime_type`. 39 | def body(request): 40 | return request.Response(body=b'\xde\xad\xbe\xef') 41 | 42 | 43 | # There exist a shortcut `json` argument. This automatically encodes the 44 | # provided object as JSON and servers it with `Content-Type` set to 45 | # `application/json; charset=utf8` 46 | def json(request): 47 | return request.Response(json={'hello': 'world'}) 48 | 49 | 50 | # You can change the default 200 status `code` for another 51 | def code(request): 52 | return request.Response(code=random.choice([200, 201, 400, 404, 500])) 53 | 54 | 55 | # And of course you can provide custom `headers`. 56 | def headers(request): 57 | return request.Response( 58 | text='headers', 59 | headers={'X-Header': 'Value', 60 | 'Refresh': '5; url=https://xkcd.com/353/'}) 61 | 62 | 63 | # Or `cookies` by using Python standard library `http.cookies.SimpleCookie`. 64 | def cookies(request): 65 | cookies = SimpleCookie() 66 | cookies['hello'] = 'world' 67 | cookies['hello']['domain'] = 'localhost' 68 | cookies['hello']['path'] = '/' 69 | cookies['hello']['max-age'] = 3600 70 | cookies['city'] = 'São Paulo' 71 | 72 | return request.Response(text='cookies', cookies=cookies) 73 | 74 | 75 | app = Application() 76 | router = app.router 77 | router.add_route('/text', text) 78 | router.add_route('/encoding', encoding) 79 | router.add_route('/mime', mime) 80 | router.add_route('/body', body) 81 | router.add_route('/json', json) 82 | router.add_route('/code', code) 83 | router.add_route('/headers', headers) 84 | router.add_route('/cookies', cookies) 85 | app.run() 86 | ``` 87 | 88 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 89 | 90 | 91 | **Next:** [Handling exceptions](6_exceptions.md) 92 | -------------------------------------------------------------------------------- /tutorial/6_exceptions.md: -------------------------------------------------------------------------------- 1 | # Handling exceptions 2 | 3 | There may be cases where you may want to respond with a custom response instead of a 500 Internal Server Error when an exception(s) is raised. Or you might want to override the default 404 handler. Enter exception handlers. 4 | 5 | ```python 6 | # examples/6_exceptions/exceptions.py 7 | from japronto import Application, RouteNotFoundException 8 | 9 | 10 | # Those are our custom exceptions we want to turn into 200 response. 11 | class KittyError(Exception): 12 | def __init__(self): 13 | self.greet = 'meow' 14 | 15 | 16 | class DoggieError(Exception): 17 | def __init__(self): 18 | self.greet = 'woof' 19 | 20 | 21 | # The two handlers below raise exceptions which will be turned 22 | # into 200 responses by the handlers registered later 23 | def cat(request): 24 | raise KittyError() 25 | 26 | 27 | def dog(request): 28 | raise DoggieError() 29 | 30 | 31 | # This handler raises ZeroDivisionError which doesnt have an error 32 | # handler registered so it will result in 500 Internal Server Error 33 | def unhandled(request): 34 | 1 / 0 35 | 36 | 37 | app = Application() 38 | 39 | r = app.router 40 | r.add_route('/cat', cat) 41 | r.add_route('/dog', dog) 42 | r.add_route('/unhandled', unhandled) 43 | 44 | 45 | # These two are handlers for `Kitty` and `DoggyError`s. 46 | def handle_cat(request, exception): 47 | return request.Response(text='Just a kitty, ' + exception.greet) 48 | 49 | 50 | def handle_dog(request, exception): 51 | return request.Response(text='Just a doggie, ' + exception.greet) 52 | 53 | 54 | # You can also override default 404 handler if you want 55 | def handle_not_found(request, exception): 56 | return request.Response(code=404, text="Are you lost, pal?") 57 | 58 | 59 | # register all the error handlers so they are actually effective 60 | app.add_error_handler(KittyError, handle_cat) 61 | app.add_error_handler(DoggieError, handle_dog) 62 | app.add_error_handler(RouteNotFoundException, handle_not_found) 63 | 64 | app.run() 65 | ``` 66 | 67 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 68 | 69 | **Next:** [Extending request](7_extend.md) 70 | -------------------------------------------------------------------------------- /tutorial/7_extend.md: -------------------------------------------------------------------------------- 1 | # Extending Request object 2 | 3 | You can register custom properties and methods on Request object. This is typically done to accommodate objects that 4 | are bound to request lifetime such as database connections, sessions or caching. This lets you easily access that logic in one place. There also exist a mechanism to execute code 5 | on request completion by registering a callback in a view. 6 | 7 | ```python 8 | # examples/7_extend/extend.py 9 | from japronto import Application 10 | 11 | 12 | # This view accesses custom method host_startswith 13 | # and a custom property reversed_agent. Both are registered later. 14 | def extended_hello(request): 15 | if request.host_startswith('api.'): 16 | text = 'Hello ' + request.reversed_agent 17 | else: 18 | text = 'Hello stranger' 19 | 20 | return request.Response(text=text) 21 | 22 | 23 | # This view registers a callback, such callbacks are executed after handler 24 | # exit and the response is ready to be sent over the wire. 25 | def with_callback(request): 26 | def cb(r): 27 | print('Done!') 28 | 29 | request.add_done_callback(cb) 30 | 31 | return request.Response(text='cb') 32 | 33 | 34 | # This is a body for reversed_agent property 35 | def reversed_agent(request): 36 | return request.headers['User-Agent'][::-1] 37 | 38 | 39 | # This is a body for host_startswith method 40 | # Custom methods and properties always accept request 41 | # object. 42 | def host_startswith(request, prefix): 43 | return request.headers['Host'].startswith(prefix) 44 | 45 | app = Application() 46 | # Finally register out custom property and method 47 | # By default the names are taken from function names 48 | # unelss you provide `name` keyword parameter. 49 | app.extend_request(reversed_agent, property=True) 50 | app.extend_request(host_startswith) 51 | 52 | r = app.router 53 | r.add_route('/', extended_hello) 54 | r.add_route('/callback', with_callback) 55 | 56 | app.run() 57 | ``` 58 | 59 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 60 | -------------------------------------------------------------------------------- /tutorial/8_template.md: -------------------------------------------------------------------------------- 1 | # Responding with HTML 2 | 3 | Serving HTML from japronto is as simple as adding a MIME type of `text/html` to the Response. Jinja2 templating can be leveraged as well, although in the meantime you will have to do the heavy lifting of rendering templates before sending in a response. 4 | 5 | Copy and paste following code into a file named `html.py`: 6 | 7 | ```python 8 | # examples/8_template/template.py 9 | from japronto import Application 10 | from jinja2 import Template 11 | 12 | 13 | # A view can read HTML from a file 14 | def index(request): 15 | with open('index.html') as html_file: 16 | return request.Response(text=html_file.read(), mime_type='text/html') 17 | 18 | 19 | # A view could also return a raw HTML string 20 | def example(request): 21 | return request.Response(text='

Some HTML!

', mime_type='text/html') 22 | 23 | 24 | # A view could also return a rendered jinja2 template 25 | def jinja(request): 26 | template = Template('

Hello {{ name }}!

') 27 | return request.Response(text=template.render(name='World'), 28 | mime_type='text/html') 29 | 30 | 31 | # Create the japronto application 32 | app = Application() 33 | 34 | # Add routes to the app 35 | app.router.add_route('/', index) 36 | app.router.add_route('/example', example) 37 | app.router.add_route('/jinja2', jinja) 38 | 39 | # Start the server 40 | app.run(debug=True) 41 | ``` 42 | 43 | The source code for all the examples can be found in [examples directory](https://github.com/squeaky-pl/japronto/tree/master/examples). 44 | 45 | --------------------------------------------------------------------------------