├── .dockerignore ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── TODO.md ├── application.py ├── bottle.py ├── demo_data.py ├── docker-compose.yml ├── redis ├── __init__.py ├── _compat.py ├── client.py ├── connection.py ├── exceptions.py ├── lock.py ├── sentinel.py └── utils.py ├── static ├── app.css ├── app.js ├── handlebars.min.js ├── helpers.js └── index.html └── waitress ├── __init__.py ├── __main__.py ├── adjustments.py ├── buffers.py ├── channel.py ├── compat.py ├── parser.py ├── receiver.py ├── runner.py ├── server.py ├── task.py ├── tests ├── __init__.py ├── fixtureapps │ ├── __init__.py │ ├── badcl.py │ ├── echo.py │ ├── error.py │ ├── filewrapper.py │ ├── getline.py │ ├── groundhog1.jpg │ ├── nocl.py │ ├── runner.py │ ├── sleepy.py │ ├── toolarge.py │ └── writecb.py ├── test_adjustments.py ├── test_buffers.py ├── test_channel.py ├── test_compat.py ├── test_functional.py ├── test_init.py ├── test_parser.py ├── test_receiver.py ├── test_regression.py ├── test_runner.py ├── test_server.py ├── test_task.py ├── test_trigger.py └── test_utilities.py ├── trigger.py └── utilities.py /.dockerignore: -------------------------------------------------------------------------------- 1 | # files 2 | .DS_Store 3 | .dockerignore 4 | .gitignore 5 | docker-compose.yml 6 | Dockerfile 7 | README.md 8 | 9 | # directories 10 | .git 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | data/ 3 | __pycache__ 4 | sandbox/ 5 | .pytest_cache/ 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | script: 5 | - pytest 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7-alpine 2 | LABEL maintainer "Mitja Felicijan " 3 | WORKDIR /usr/src/app 4 | COPY . . 5 | CMD ["python", "./application.py"] 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Mitja Felicijan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to use, 8 | copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the 9 | Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REPO_NAME = mitjafelicijan/redis-marshal 2 | 3 | dev: 4 | find -type f | egrep -i "*.py|*.yml" | entr -r python2 application.py --debug 5 | 6 | clean: 7 | find . -type f -name '*.pyc' -delete 8 | 9 | docker-build: 10 | docker build -t $(REPO_NAME):latest . 11 | 12 | docker-publish: 13 | docker tag $(REPO_NAME):latest $(REPO_NAME) 14 | docker push $(REPO_NAME) 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Redis Marshal 2 | 3 | [![Build Status MASTER](https://travis-ci.org/mitjafelicijan/redis-marshal.svg?branch=master)](https://travis-ci.org/mitjafelicijan/redis-marshal) [![Build Status DEVEL](https://travis-ci.org/mitjafelicijan/redis-marshal.svg?branch=devel)](https://travis-ci.org/mitjafelicijan/redis-marshal) 4 | 5 | ![Query window](https://user-images.githubusercontent.com/296714/40789890-d25e23dc-64f3-11e8-8667-d20fff6ee493.png) 6 | 7 | 8 | #### Additional resources 9 | 10 | - [Screencast on Youtube](https://www.youtube.com/watch?v=KQVR3dXxGUQ) 11 | - [Add suggestions or issues](https://github.com/mitjafelicijan/redis-marshal/issues/new) 12 | 13 | 14 | ### Current data type support 15 | 16 | - [x] Strings 17 | - [ ] Lists 18 | - [ ] Sets 19 | - [x] Hashes 20 | - [ ] Sorted sets 21 | - [ ] Bitmaps and HyperLogLogs 22 | 23 | Todo list is located in [TODO.md](TODO.md). 24 | 25 | 26 | ### Software characteristics 27 | 28 | - Written in Python2.7 (Python3.x fails on JSON serialiation - working on it). 29 | - All dependencies are included with source (no pip or virtualenv needed). 30 | - Works with Docker and docker-compose. 31 | - No autentication implemented (use Nginx or Caddy as a reverse proxy and add Basic-Auth). 32 | - Allows bulk key deletion. 33 | - Allows table sorting of results based on type, key and ttl. 34 | - Autogenerates forms from hashsets and enables adding and removing attributes. 35 | - Allows executing commands. 36 | 37 | 38 | ### Supported glob-style patterns 39 | 40 | - h?llo matches hello, hallo and hxllo 41 | - h*llo matches hllo and heeeello 42 | - h[ae]llo matches hello and hallo, but not hillo 43 | - h[^e]llo matches hallo, hbllo, ... but not hello 44 | - h[a-b]llo matches hallo and hbllo 45 | 46 | Use `\` to escape special characters if you want to match them verbatim. More on https://redis.io/commands/keys. 47 | 48 | 49 | 50 | ### Application arguments 51 | 52 | ``` 53 | usage: application.py [-h] [--port PORT] [--host HOST] [--production] 54 | [--debug] [--reloader] [--path PATH] 55 | [--redis-host REDIS_HOST] [--redis-port REDIS_PORT] 56 | [--redis-db REDIS_DATABASE] 57 | 58 | optional arguments: 59 | -h, --help show this help message and exit 60 | --port PORT server port 61 | --host HOST server host 62 | --production enables production mode and reduces logging 63 | --debug application in debug mode 64 | --reloader application reloads on source change 65 | --path PATH Path of app / or /marshal/ 66 | --redis-host REDIS_HOST 67 | Redis host 68 | --redis-port REDIS_PORT 69 | Redis port 70 | --redis-db REDIS_DATABASE 71 | Redis database number 72 | ``` 73 | 74 | 75 | ### Using with Docker (docker-compose) 76 | 77 | ```sh 78 | docker-compose up 79 | ``` 80 | 81 | ```yaml 82 | version: "3" 83 | networks: 84 | redis-marshal-net: 85 | driver: bridge 86 | services: 87 | redis-marshal: 88 | image: mitjafelicijan/redis-marshal 89 | ports: 90 | - "9001:9001" 91 | depends_on: 92 | - redis 93 | command: python ./application.py --port 9001 --redis-host redis-marshal.internal 94 | networks: 95 | - redis-marshal-net 96 | redis: 97 | image: redis 98 | hostname: redis 99 | ports: 100 | - "6379:6379" 101 | command: /usr/local/bin/redis-server --appendonly yes --appendfilename history.aof 102 | volumes: 103 | - ${PWD}/data:/data 104 | networks: 105 | redis-marshal-net: 106 | aliases: 107 | - redis-marshal.internal 108 | ``` 109 | 110 | 111 | ### Using on local machine 112 | 113 | Before running start Redis Server on local machine. 114 | 115 | ```sh 116 | wget https://github.com/mitjafelicijan/redis-marshal/archive/master.zip 117 | unzip master.zip 118 | cd redis-marshal-master 119 | python2 application.py 120 | ``` 121 | 122 | 123 | ### Made with the help of 124 | 125 | - https://github.com/bottlepy/bottle 126 | - https://github.com/Pylons/waitress 127 | - https://github.com/andymccurdy/redis-py 128 | - https://github.com/wycats/handlebars.js/ 129 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # High priority: 2 | 3 | - [ ] add list support 4 | - [ ] add sets support 5 | - [ ] add sorted sets support 6 | - [ ] add bitmaps and hyperloglogs support 7 | 8 | - [ ] add query took ms in results 9 | - [x] bulk delete sends command for deletetion in batch of 10 10 | - [ ] add connection indicator with redis ping and add try catch 11 | - [x] when usign * for query check num_keys and if larger that 500 alert user or even disable query 12 | - [x] on query enter press disable input and on results reenable input... loading 13 | - [x] proccessed num format into human friendly 14 | - [ ] add execute command 15 | 16 | # Low priority: 17 | 18 | - [ ] add authentication (login form) with bottlepy 19 | -------------------------------------------------------------------------------- /application.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | import datetime 6 | import json 7 | import redis 8 | import logging 9 | import argparse 10 | import bottle 11 | 12 | CACHE_VER = "20180531" 13 | MAX_SIZE = 300 14 | 15 | # terminal arguments 16 | ap = argparse.ArgumentParser() 17 | ap.add_argument("--port", dest="port", default=5000, type=int, help="server port") 18 | ap.add_argument("--host", dest="host", default="0.0.0.0", help="server host") 19 | ap.add_argument("--production", dest="production", action="store_true", help="enables production mode and reduces logging") 20 | ap.add_argument("--debug", dest="debug", action="store_true", help="application in debug mode") 21 | ap.add_argument("--reloader", dest="reloader", action="store_true", help="application reloads on source change") 22 | ap.add_argument("--path", dest="path", default="/", help="Path of app / or /marshal/") 23 | ap.add_argument("--redis-host", dest="redis_host", default="localhost", help="Redis host") 24 | ap.add_argument("--redis-port", dest="redis_port", type=int, default=6379, help="Redis port") 25 | ap.add_argument("--redis-db", dest="redis_database", type=int, default=0, help="Redis database number") 26 | args = vars(ap.parse_args()) 27 | 28 | 29 | # setting up logging based on prod/devel stage 30 | if args["production"]: 31 | log_level = logging.CRITICAL 32 | else: 33 | log_level = logging.DEBUG 34 | 35 | logging.basicConfig(stream=sys.stdout, level=log_level, format="%(asctime)s %(levelname)s %(name)s -> %(message)s") 36 | 37 | 38 | # setting up application 39 | app = application = bottle.Bottle() 40 | app.config["r"] = redis.StrictRedis(host=args["redis_host"], port=args["redis_port"], db=args["redis_database"]) 41 | 42 | 43 | @app.route("{}static/".format(args["path"])) 44 | def send_static(filename): 45 | return bottle.static_file(filename, root = str(os.getcwd()) + "/static") 46 | 47 | 48 | @app.route("{}".format(args["path"]), method=["GET"]) 49 | def route_default(): 50 | with open("static/index.html", "r") as fp: 51 | data = str(fp.read()) 52 | data = data.replace("$$path$$", args["path"]) 53 | data = data.replace("$$cache$$", CACHE_VER) 54 | data = data.replace("$$db$$", str(args["redis_database"])) 55 | return data 56 | 57 | @app.route("{}api/info".format(args["path"]), method=["GET"]) 58 | def route_api_info(): 59 | bottle.response.headers["Content-Type"] = "application/json" 60 | bottle.response.headers["Cache-Control"] = "no-cache" 61 | return json.dumps(app.config["r"].info()) 62 | 63 | @app.route("{}api/scan".format(args["path"]), method=["GET"]) 64 | def route_api_scan(): 65 | payload = { "limitReached": False, "items": [], "elapsed": 0 } 66 | query = bottle.request.query.q 67 | pattern = query if query != "" else "*" 68 | key_counter = 0 69 | start = datetime.datetime.now() 70 | for key in app.config["r"].scan_iter(pattern): 71 | if key_counter < MAX_SIZE: 72 | payload["items"].append({ 73 | "key": key, 74 | "ttl": app.config["r"].ttl(key), 75 | "type": app.config["r"].type(key), 76 | }) 77 | key_counter += 1 78 | else: 79 | payload["limitReached"] = True 80 | break 81 | end = datetime.datetime.now() 82 | diff = (end - start) 83 | elapsed_ms = (diff.days * 86400000) + (diff.seconds * 1000) + (diff.microseconds / 1000) 84 | payload["elapsed"] = elapsed_ms 85 | 86 | bottle.response.headers["Content-Type"] = "application/json" 87 | bottle.response.headers["Cache-Control"] = "no-cache" 88 | return json.dumps(payload) 89 | 90 | @app.route("{}api/del".format(args["path"]), method=["POST"]) 91 | def route_api_del(): 92 | raw = bottle.request.body.read() 93 | response = { "status": False, "items": [] } 94 | payload = json.loads(raw) 95 | if len(payload["items"]) > 0: 96 | for key in payload["items"]: 97 | try: 98 | app.config["r"].delete(key) 99 | response["items"].append(key) 100 | except Exception as e: 101 | print (e) 102 | response["status"] = True 103 | 104 | bottle.response.headers["Content-Type"] = "application/json" 105 | bottle.response.headers["Cache-Control"] = "no-cache" 106 | return json.dumps(response) 107 | 108 | @app.route("{}api/get".format(args["path"]), method=["GET"]) 109 | def route_api_get(): 110 | key = bottle.request.query.key 111 | key_type = bottle.request.query.type 112 | response = { "key": key, "type": key_type } 113 | if key != "": 114 | if key_type == "string": 115 | response["value"] = app.config["r"].get(key) 116 | elif key_type == "hash": 117 | response["value"] = app.config["r"].hgetall(key) 118 | bottle.response.headers["Content-Type"] = "application/json" 119 | bottle.response.headers["Cache-Control"] = "no-cache" 120 | return json.dumps(response) 121 | 122 | @app.route("{}api/set".format(args["path"]), method=["POST"]) 123 | def route_api_set(): 124 | raw = bottle.request.body.read() 125 | response = { "status": False } 126 | payload = json.loads(raw) 127 | 128 | try: 129 | # delete old key 130 | app.config["r"].delete(payload["key"]) 131 | 132 | if payload["type"] == "hash": 133 | for item in payload["value"]: 134 | app.config["r"].hset(payload["key"], item["key"], item["value"]) 135 | 136 | elif payload["type"] == "string": 137 | app.config["r"].set(payload["key"], payload["value"]) 138 | 139 | response["status"] = True 140 | except Exception as e: 141 | print (e) 142 | 143 | bottle.response.headers["Content-Type"] = "application/json" 144 | bottle.response.headers["Cache-Control"] = "no-cache" 145 | return json.dumps(response) 146 | 147 | @app.route("{}api/exec".format(args["path"]), method=["POST"]) 148 | def route_api_exec(): 149 | raw = bottle.request.body.read() 150 | response = { "status": False, "message": None } 151 | payload = json.loads(raw) 152 | if "cmd" in payload: 153 | try: 154 | response["message"] = app.config["r"].execute_command(payload["cmd"]) 155 | response["status"] = True 156 | except Exception as e: 157 | response["message"] = str(e) 158 | print (e) 159 | 160 | bottle.response.headers["Content-Type"] = "application/json" 161 | bottle.response.headers["Cache-Control"] = "no-cache" 162 | return json.dumps(response) 163 | 164 | 165 | 166 | # starting server 167 | if __name__ == "__main__": 168 | bottle.run( 169 | app = app, 170 | server = "waitress", 171 | host = args["host"], 172 | port = args["port"], 173 | debug = args["debug"], 174 | reloader = args["reloader"], 175 | ) 176 | -------------------------------------------------------------------------------- /demo_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # inserts demo testing data for development 3 | # for i in {1..50}; do python2 demo_data.py; done 4 | 5 | import random 6 | import string 7 | import argparse 8 | import redis 9 | 10 | from faker import Faker 11 | fake = Faker() 12 | 13 | ap = argparse.ArgumentParser() 14 | ap.add_argument("--num-records", dest="num_records", default=50, type=int, help="number of demo records per type") 15 | ap.add_argument("--redis-host", dest="redis_host", default="localhost", help="Redis host") 16 | ap.add_argument("--redis-port", dest="redis_port", type=int, default=6379, help="Redis port") 17 | ap.add_argument("--redis-db", dest="redis_database", type=int, default=0, help="Redis database number") 18 | args = vars(ap.parse_args()) 19 | 20 | r = redis.StrictRedis(host=args["redis_host"], port=args["redis_port"], db=args["redis_database"]) 21 | 22 | print("inserting {} {} records ...".format(args["num_records"], "string")) 23 | for i in xrange(args["num_records"]): 24 | uid = "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(20)) 25 | r.set("demo_{}_{}".format("string", uid), fake.text()) 26 | 27 | print("inserting {} {} records ...".format(args["num_records"], "hash")) 28 | for i in xrange(args["num_records"]): 29 | uid = "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(20)) 30 | r.hset("demo_{}_{}".format("users", uid), "name", fake.name()) 31 | r.hset("demo_{}_{}".format("users", uid), "address", fake.address()) 32 | r.hset("demo_{}_{}".format("users", uid), "job", fake.job()) 33 | r.hset("demo_{}_{}".format("users", uid), "phone_number", fake.phone_number()) 34 | r.hset("demo_{}_{}".format("users", uid), "company", fake.company()) 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | networks: 4 | redis-marshal-net: 5 | driver: bridge 6 | 7 | services: 8 | 9 | redis-marshal: 10 | image: mitjafelicijan/redis-marshal 11 | ports: 12 | - "9001:9001" 13 | depends_on: 14 | - redis 15 | command: python ./application.py --port 9001 --redis-host redis-marshal.internal 16 | networks: 17 | - redis-marshal-net 18 | 19 | redis: 20 | image: redis 21 | hostname: redis 22 | ports: 23 | - "6379:6379" 24 | command: /usr/local/bin/redis-server --appendonly yes --appendfilename history.aof 25 | volumes: 26 | - ${PWD}/data:/data 27 | networks: 28 | redis-marshal-net: 29 | aliases: 30 | - redis-marshal.internal 31 | -------------------------------------------------------------------------------- /redis/__init__.py: -------------------------------------------------------------------------------- 1 | from redis.client import Redis, StrictRedis 2 | from redis.connection import ( 3 | BlockingConnectionPool, 4 | ConnectionPool, 5 | Connection, 6 | SSLConnection, 7 | UnixDomainSocketConnection 8 | ) 9 | from redis.utils import from_url 10 | from redis.exceptions import ( 11 | AuthenticationError, 12 | BusyLoadingError, 13 | ConnectionError, 14 | DataError, 15 | InvalidResponse, 16 | PubSubError, 17 | ReadOnlyError, 18 | RedisError, 19 | ResponseError, 20 | TimeoutError, 21 | WatchError 22 | ) 23 | 24 | 25 | __version__ = '2.10.6' 26 | VERSION = tuple(map(int, __version__.split('.'))) 27 | 28 | __all__ = [ 29 | 'Redis', 'StrictRedis', 'ConnectionPool', 'BlockingConnectionPool', 30 | 'Connection', 'SSLConnection', 'UnixDomainSocketConnection', 'from_url', 31 | 'AuthenticationError', 'BusyLoadingError', 'ConnectionError', 'DataError', 32 | 'InvalidResponse', 'PubSubError', 'ReadOnlyError', 'RedisError', 33 | 'ResponseError', 'TimeoutError', 'WatchError' 34 | ] 35 | -------------------------------------------------------------------------------- /redis/_compat.py: -------------------------------------------------------------------------------- 1 | """Internal module for Python 2 backwards compatibility.""" 2 | import errno 3 | import sys 4 | 5 | try: 6 | InterruptedError = InterruptedError 7 | except: 8 | InterruptedError = OSError 9 | 10 | # For Python older than 3.5, retry EINTR. 11 | if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and 12 | sys.version_info[1] < 5): 13 | # Adapted from https://bugs.python.org/review/23863/patch/14532/54418 14 | import socket 15 | import time 16 | import errno 17 | 18 | from select import select as _select 19 | 20 | def select(rlist, wlist, xlist, timeout): 21 | while True: 22 | try: 23 | return _select(rlist, wlist, xlist, timeout) 24 | except InterruptedError as e: 25 | # Python 2 does not define InterruptedError, instead 26 | # try to catch an OSError with errno == EINTR == 4. 27 | if getattr(e, 'errno', None) == getattr(errno, 'EINTR', 4): 28 | continue 29 | raise 30 | 31 | # Wrapper for handling interruptable system calls. 32 | def _retryable_call(s, func, *args, **kwargs): 33 | # Some modules (SSL) use the _fileobject wrapper directly and 34 | # implement a smaller portion of the socket interface, thus we 35 | # need to let them continue to do so. 36 | timeout, deadline = None, 0.0 37 | attempted = False 38 | try: 39 | timeout = s.gettimeout() 40 | except AttributeError: 41 | pass 42 | 43 | if timeout: 44 | deadline = time.time() + timeout 45 | 46 | try: 47 | while True: 48 | if attempted and timeout: 49 | now = time.time() 50 | if now >= deadline: 51 | raise socket.error(errno.EWOULDBLOCK, "timed out") 52 | else: 53 | # Overwrite the timeout on the socket object 54 | # to take into account elapsed time. 55 | s.settimeout(deadline - now) 56 | try: 57 | attempted = True 58 | return func(*args, **kwargs) 59 | except socket.error as e: 60 | if e.args[0] == errno.EINTR: 61 | continue 62 | raise 63 | finally: 64 | # Set the existing timeout back for future 65 | # calls. 66 | if timeout: 67 | s.settimeout(timeout) 68 | 69 | def recv(sock, *args, **kwargs): 70 | return _retryable_call(sock, sock.recv, *args, **kwargs) 71 | 72 | def recv_into(sock, *args, **kwargs): 73 | return _retryable_call(sock, sock.recv_into, *args, **kwargs) 74 | 75 | else: # Python 3.5 and above automatically retry EINTR 76 | from select import select 77 | 78 | def recv(sock, *args, **kwargs): 79 | return sock.recv(*args, **kwargs) 80 | 81 | def recv_into(sock, *args, **kwargs): 82 | return sock.recv_into(*args, **kwargs) 83 | 84 | if sys.version_info[0] < 3: 85 | from urllib import unquote 86 | from urlparse import parse_qs, urlparse 87 | from itertools import imap, izip 88 | from string import letters as ascii_letters 89 | from Queue import Queue 90 | try: 91 | from cStringIO import StringIO as BytesIO 92 | except ImportError: 93 | from StringIO import StringIO as BytesIO 94 | 95 | # special unicode handling for python2 to avoid UnicodeDecodeError 96 | def safe_unicode(obj, *args): 97 | """ return the unicode representation of obj """ 98 | try: 99 | return unicode(obj, *args) 100 | except UnicodeDecodeError: 101 | # obj is byte string 102 | ascii_text = str(obj).encode('string_escape') 103 | return unicode(ascii_text) 104 | 105 | def iteritems(x): 106 | return x.iteritems() 107 | 108 | def iterkeys(x): 109 | return x.iterkeys() 110 | 111 | def itervalues(x): 112 | return x.itervalues() 113 | 114 | def nativestr(x): 115 | return x if isinstance(x, str) else x.encode('utf-8', 'replace') 116 | 117 | def u(x): 118 | return x.decode() 119 | 120 | def b(x): 121 | return x 122 | 123 | def next(x): 124 | return x.next() 125 | 126 | def byte_to_chr(x): 127 | return x 128 | 129 | unichr = unichr 130 | xrange = xrange 131 | basestring = basestring 132 | unicode = unicode 133 | bytes = str 134 | long = long 135 | else: 136 | from urllib.parse import parse_qs, unquote, urlparse 137 | from io import BytesIO 138 | from string import ascii_letters 139 | from queue import Queue 140 | 141 | def iteritems(x): 142 | return iter(x.items()) 143 | 144 | def iterkeys(x): 145 | return iter(x.keys()) 146 | 147 | def itervalues(x): 148 | return iter(x.values()) 149 | 150 | def byte_to_chr(x): 151 | return chr(x) 152 | 153 | def nativestr(x): 154 | return x if isinstance(x, str) else x.decode('utf-8', 'replace') 155 | 156 | def u(x): 157 | return x 158 | 159 | def b(x): 160 | return x.encode('latin-1') if not isinstance(x, bytes) else x 161 | 162 | next = next 163 | unichr = chr 164 | imap = map 165 | izip = zip 166 | xrange = range 167 | basestring = str 168 | unicode = str 169 | safe_unicode = str 170 | bytes = bytes 171 | long = int 172 | 173 | try: # Python 3 174 | from queue import LifoQueue, Empty, Full 175 | except ImportError: 176 | from Queue import Empty, Full 177 | try: # Python 2.6 - 2.7 178 | from Queue import LifoQueue 179 | except ImportError: # Python 2.5 180 | from Queue import Queue 181 | # From the Python 2.7 lib. Python 2.5 already extracted the core 182 | # methods to aid implementating different queue organisations. 183 | 184 | class LifoQueue(Queue): 185 | "Override queue methods to implement a last-in first-out queue." 186 | 187 | def _init(self, maxsize): 188 | self.maxsize = maxsize 189 | self.queue = [] 190 | 191 | def _qsize(self, len=len): 192 | return len(self.queue) 193 | 194 | def _put(self, item): 195 | self.queue.append(item) 196 | 197 | def _get(self): 198 | return self.queue.pop() 199 | -------------------------------------------------------------------------------- /redis/exceptions.py: -------------------------------------------------------------------------------- 1 | "Core exceptions raised by the Redis client" 2 | from redis._compat import unicode 3 | 4 | 5 | class RedisError(Exception): 6 | pass 7 | 8 | 9 | # python 2.5 doesn't implement Exception.__unicode__. Add it here to all 10 | # our exception types 11 | if not hasattr(RedisError, '__unicode__'): 12 | def __unicode__(self): 13 | if isinstance(self.args[0], unicode): 14 | return self.args[0] 15 | return unicode(self.args[0]) 16 | RedisError.__unicode__ = __unicode__ 17 | 18 | 19 | class AuthenticationError(RedisError): 20 | pass 21 | 22 | 23 | class ConnectionError(RedisError): 24 | pass 25 | 26 | 27 | class TimeoutError(RedisError): 28 | pass 29 | 30 | 31 | class BusyLoadingError(ConnectionError): 32 | pass 33 | 34 | 35 | class InvalidResponse(RedisError): 36 | pass 37 | 38 | 39 | class ResponseError(RedisError): 40 | pass 41 | 42 | 43 | class DataError(RedisError): 44 | pass 45 | 46 | 47 | class PubSubError(RedisError): 48 | pass 49 | 50 | 51 | class WatchError(RedisError): 52 | pass 53 | 54 | 55 | class NoScriptError(ResponseError): 56 | pass 57 | 58 | 59 | class ExecAbortError(ResponseError): 60 | pass 61 | 62 | 63 | class ReadOnlyError(ResponseError): 64 | pass 65 | 66 | 67 | class LockError(RedisError, ValueError): 68 | "Errors acquiring or releasing a lock" 69 | # NOTE: For backwards compatability, this class derives from ValueError. 70 | # This was originally chosen to behave like threading.Lock. 71 | pass 72 | -------------------------------------------------------------------------------- /redis/lock.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time as mod_time 3 | import uuid 4 | from redis.exceptions import LockError, WatchError 5 | from redis.utils import dummy 6 | from redis._compat import b 7 | 8 | 9 | class Lock(object): 10 | """ 11 | A shared, distributed Lock. Using Redis for locking allows the Lock 12 | to be shared across processes and/or machines. 13 | 14 | It's left to the user to resolve deadlock issues and make sure 15 | multiple clients play nicely together. 16 | """ 17 | def __init__(self, redis, name, timeout=None, sleep=0.1, 18 | blocking=True, blocking_timeout=None, thread_local=True): 19 | """ 20 | Create a new Lock instance named ``name`` using the Redis client 21 | supplied by ``redis``. 22 | 23 | ``timeout`` indicates a maximum life for the lock. 24 | By default, it will remain locked until release() is called. 25 | ``timeout`` can be specified as a float or integer, both representing 26 | the number of seconds to wait. 27 | 28 | ``sleep`` indicates the amount of time to sleep per loop iteration 29 | when the lock is in blocking mode and another client is currently 30 | holding the lock. 31 | 32 | ``blocking`` indicates whether calling ``acquire`` should block until 33 | the lock has been acquired or to fail immediately, causing ``acquire`` 34 | to return False and the lock not being acquired. Defaults to True. 35 | Note this value can be overridden by passing a ``blocking`` 36 | argument to ``acquire``. 37 | 38 | ``blocking_timeout`` indicates the maximum amount of time in seconds to 39 | spend trying to acquire the lock. A value of ``None`` indicates 40 | continue trying forever. ``blocking_timeout`` can be specified as a 41 | float or integer, both representing the number of seconds to wait. 42 | 43 | ``thread_local`` indicates whether the lock token is placed in 44 | thread-local storage. By default, the token is placed in thread local 45 | storage so that a thread only sees its token, not a token set by 46 | another thread. Consider the following timeline: 47 | 48 | time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds. 49 | thread-1 sets the token to "abc" 50 | time: 1, thread-2 blocks trying to acquire `my-lock` using the 51 | Lock instance. 52 | time: 5, thread-1 has not yet completed. redis expires the lock 53 | key. 54 | time: 5, thread-2 acquired `my-lock` now that it's available. 55 | thread-2 sets the token to "xyz" 56 | time: 6, thread-1 finishes its work and calls release(). if the 57 | token is *not* stored in thread local storage, then 58 | thread-1 would see the token value as "xyz" and would be 59 | able to successfully release the thread-2's lock. 60 | 61 | In some use cases it's necessary to disable thread local storage. For 62 | example, if you have code where one thread acquires a lock and passes 63 | that lock instance to a worker thread to release later. If thread 64 | local storage isn't disabled in this case, the worker thread won't see 65 | the token set by the thread that acquired the lock. Our assumption 66 | is that these cases aren't common and as such default to using 67 | thread local storage. 68 | """ 69 | self.redis = redis 70 | self.name = name 71 | self.timeout = timeout 72 | self.sleep = sleep 73 | self.blocking = blocking 74 | self.blocking_timeout = blocking_timeout 75 | self.thread_local = bool(thread_local) 76 | self.local = threading.local() if self.thread_local else dummy() 77 | self.local.token = None 78 | if self.timeout and self.sleep > self.timeout: 79 | raise LockError("'sleep' must be less than 'timeout'") 80 | 81 | def __enter__(self): 82 | # force blocking, as otherwise the user would have to check whether 83 | # the lock was actually acquired or not. 84 | self.acquire(blocking=True) 85 | return self 86 | 87 | def __exit__(self, exc_type, exc_value, traceback): 88 | self.release() 89 | 90 | def acquire(self, blocking=None, blocking_timeout=None): 91 | """ 92 | Use Redis to hold a shared, distributed lock named ``name``. 93 | Returns True once the lock is acquired. 94 | 95 | If ``blocking`` is False, always return immediately. If the lock 96 | was acquired, return True, otherwise return False. 97 | 98 | ``blocking_timeout`` specifies the maximum number of seconds to 99 | wait trying to acquire the lock. 100 | """ 101 | sleep = self.sleep 102 | token = b(uuid.uuid1().hex) 103 | if blocking is None: 104 | blocking = self.blocking 105 | if blocking_timeout is None: 106 | blocking_timeout = self.blocking_timeout 107 | stop_trying_at = None 108 | if blocking_timeout is not None: 109 | stop_trying_at = mod_time.time() + blocking_timeout 110 | while 1: 111 | if self.do_acquire(token): 112 | self.local.token = token 113 | return True 114 | if not blocking: 115 | return False 116 | if stop_trying_at is not None and mod_time.time() > stop_trying_at: 117 | return False 118 | mod_time.sleep(sleep) 119 | 120 | def do_acquire(self, token): 121 | if self.timeout: 122 | # convert to milliseconds 123 | timeout = int(self.timeout * 1000) 124 | else: 125 | timeout = None 126 | if self.redis.set(self.name, token, nx=True, px=timeout): 127 | return True 128 | return False 129 | 130 | def release(self): 131 | "Releases the already acquired lock" 132 | expected_token = self.local.token 133 | if expected_token is None: 134 | raise LockError("Cannot release an unlocked lock") 135 | self.local.token = None 136 | self.do_release(expected_token) 137 | 138 | def do_release(self, expected_token): 139 | name = self.name 140 | 141 | def execute_release(pipe): 142 | lock_value = pipe.get(name) 143 | if lock_value != expected_token: 144 | raise LockError("Cannot release a lock that's no longer owned") 145 | pipe.delete(name) 146 | 147 | self.redis.transaction(execute_release, name) 148 | 149 | def extend(self, additional_time): 150 | """ 151 | Adds more time to an already acquired lock. 152 | 153 | ``additional_time`` can be specified as an integer or a float, both 154 | representing the number of seconds to add. 155 | """ 156 | if self.local.token is None: 157 | raise LockError("Cannot extend an unlocked lock") 158 | if self.timeout is None: 159 | raise LockError("Cannot extend a lock with no timeout") 160 | return self.do_extend(additional_time) 161 | 162 | def do_extend(self, additional_time): 163 | pipe = self.redis.pipeline() 164 | pipe.watch(self.name) 165 | lock_value = pipe.get(self.name) 166 | if lock_value != self.local.token: 167 | raise LockError("Cannot extend a lock that's no longer owned") 168 | expiration = pipe.pttl(self.name) 169 | if expiration is None or expiration < 0: 170 | # Redis evicted the lock key between the previous get() and now 171 | # we'll handle this when we call pexpire() 172 | expiration = 0 173 | pipe.multi() 174 | pipe.pexpire(self.name, expiration + int(additional_time * 1000)) 175 | 176 | try: 177 | response = pipe.execute() 178 | except WatchError: 179 | # someone else acquired the lock 180 | raise LockError("Cannot extend a lock that's no longer owned") 181 | if not response[0]: 182 | # pexpire returns False if the key doesn't exist 183 | raise LockError("Cannot extend a lock that's no longer owned") 184 | return True 185 | 186 | 187 | class LuaLock(Lock): 188 | """ 189 | A lock implementation that uses Lua scripts rather than pipelines 190 | and watches. 191 | """ 192 | lua_release = None 193 | lua_extend = None 194 | 195 | # KEYS[1] - lock name 196 | # ARGS[1] - token 197 | # return 1 if the lock was released, otherwise 0 198 | LUA_RELEASE_SCRIPT = """ 199 | local token = redis.call('get', KEYS[1]) 200 | if not token or token ~= ARGV[1] then 201 | return 0 202 | end 203 | redis.call('del', KEYS[1]) 204 | return 1 205 | """ 206 | 207 | # KEYS[1] - lock name 208 | # ARGS[1] - token 209 | # ARGS[2] - additional milliseconds 210 | # return 1 if the locks time was extended, otherwise 0 211 | LUA_EXTEND_SCRIPT = """ 212 | local token = redis.call('get', KEYS[1]) 213 | if not token or token ~= ARGV[1] then 214 | return 0 215 | end 216 | local expiration = redis.call('pttl', KEYS[1]) 217 | if not expiration then 218 | expiration = 0 219 | end 220 | if expiration < 0 then 221 | return 0 222 | end 223 | redis.call('pexpire', KEYS[1], expiration + ARGV[2]) 224 | return 1 225 | """ 226 | 227 | def __init__(self, *args, **kwargs): 228 | super(LuaLock, self).__init__(*args, **kwargs) 229 | LuaLock.register_scripts(self.redis) 230 | 231 | @classmethod 232 | def register_scripts(cls, redis): 233 | if cls.lua_release is None: 234 | cls.lua_release = redis.register_script(cls.LUA_RELEASE_SCRIPT) 235 | if cls.lua_extend is None: 236 | cls.lua_extend = redis.register_script(cls.LUA_EXTEND_SCRIPT) 237 | 238 | def do_release(self, expected_token): 239 | if not bool(self.lua_release(keys=[self.name], 240 | args=[expected_token], 241 | client=self.redis)): 242 | raise LockError("Cannot release a lock that's no longer owned") 243 | 244 | def do_extend(self, additional_time): 245 | additional_time = int(additional_time * 1000) 246 | if not bool(self.lua_extend(keys=[self.name], 247 | args=[self.local.token, additional_time], 248 | client=self.redis)): 249 | raise LockError("Cannot extend a lock that's no longer owned") 250 | return True 251 | -------------------------------------------------------------------------------- /redis/sentinel.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import weakref 4 | 5 | from redis.client import StrictRedis 6 | from redis.connection import ConnectionPool, Connection 7 | from redis.exceptions import (ConnectionError, ResponseError, ReadOnlyError, 8 | TimeoutError) 9 | from redis._compat import iteritems, nativestr, xrange 10 | 11 | 12 | class MasterNotFoundError(ConnectionError): 13 | pass 14 | 15 | 16 | class SlaveNotFoundError(ConnectionError): 17 | pass 18 | 19 | 20 | class SentinelManagedConnection(Connection): 21 | def __init__(self, **kwargs): 22 | self.connection_pool = kwargs.pop('connection_pool') 23 | super(SentinelManagedConnection, self).__init__(**kwargs) 24 | 25 | def __repr__(self): 26 | pool = self.connection_pool 27 | s = '%s' % (type(self).__name__, pool.service_name) 28 | if self.host: 29 | host_info = ',host=%s,port=%s' % (self.host, self.port) 30 | s = s % host_info 31 | return s 32 | 33 | def connect_to(self, address): 34 | self.host, self.port = address 35 | super(SentinelManagedConnection, self).connect() 36 | if self.connection_pool.check_connection: 37 | self.send_command('PING') 38 | if nativestr(self.read_response()) != 'PONG': 39 | raise ConnectionError('PING failed') 40 | 41 | def connect(self): 42 | if self._sock: 43 | return # already connected 44 | if self.connection_pool.is_master: 45 | self.connect_to(self.connection_pool.get_master_address()) 46 | else: 47 | for slave in self.connection_pool.rotate_slaves(): 48 | try: 49 | return self.connect_to(slave) 50 | except ConnectionError: 51 | continue 52 | raise SlaveNotFoundError # Never be here 53 | 54 | def read_response(self): 55 | try: 56 | return super(SentinelManagedConnection, self).read_response() 57 | except ReadOnlyError: 58 | if self.connection_pool.is_master: 59 | # When talking to a master, a ReadOnlyError when likely 60 | # indicates that the previous master that we're still connected 61 | # to has been demoted to a slave and there's a new master. 62 | # calling disconnect will force the connection to re-query 63 | # sentinel during the next connect() attempt. 64 | self.disconnect() 65 | raise ConnectionError('The previous master is now a slave') 66 | raise 67 | 68 | 69 | class SentinelConnectionPool(ConnectionPool): 70 | """ 71 | Sentinel backed connection pool. 72 | 73 | If ``check_connection`` flag is set to True, SentinelManagedConnection 74 | sends a PING command right after establishing the connection. 75 | """ 76 | 77 | def __init__(self, service_name, sentinel_manager, **kwargs): 78 | kwargs['connection_class'] = kwargs.get( 79 | 'connection_class', SentinelManagedConnection) 80 | self.is_master = kwargs.pop('is_master', True) 81 | self.check_connection = kwargs.pop('check_connection', False) 82 | super(SentinelConnectionPool, self).__init__(**kwargs) 83 | self.connection_kwargs['connection_pool'] = weakref.proxy(self) 84 | self.service_name = service_name 85 | self.sentinel_manager = sentinel_manager 86 | 87 | def __repr__(self): 88 | return "%s>> from redis.sentinel import Sentinel 145 | >>> sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1) 146 | >>> master = sentinel.master_for('mymaster', socket_timeout=0.1) 147 | >>> master.set('foo', 'bar') 148 | >>> slave = sentinel.slave_for('mymaster', socket_timeout=0.1) 149 | >>> slave.get('foo') 150 | 'bar' 151 | 152 | ``sentinels`` is a list of sentinel nodes. Each node is represented by 153 | a pair (hostname, port). 154 | 155 | ``min_other_sentinels`` defined a minimum number of peers for a sentinel. 156 | When querying a sentinel, if it doesn't meet this threshold, responses 157 | from that sentinel won't be considered valid. 158 | 159 | ``sentinel_kwargs`` is a dictionary of connection arguments used when 160 | connecting to sentinel instances. Any argument that can be passed to 161 | a normal Redis connection can be specified here. If ``sentinel_kwargs`` is 162 | not specified, any socket_timeout and socket_keepalive options specified 163 | in ``connection_kwargs`` will be used. 164 | 165 | ``connection_kwargs`` are keyword arguments that will be used when 166 | establishing a connection to a Redis server. 167 | """ 168 | 169 | def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None, 170 | **connection_kwargs): 171 | # if sentinel_kwargs isn't defined, use the socket_* options from 172 | # connection_kwargs 173 | if sentinel_kwargs is None: 174 | sentinel_kwargs = dict([(k, v) 175 | for k, v in iteritems(connection_kwargs) 176 | if k.startswith('socket_') 177 | ]) 178 | self.sentinel_kwargs = sentinel_kwargs 179 | 180 | self.sentinels = [StrictRedis(hostname, port, **self.sentinel_kwargs) 181 | for hostname, port in sentinels] 182 | self.min_other_sentinels = min_other_sentinels 183 | self.connection_kwargs = connection_kwargs 184 | 185 | def __repr__(self): 186 | sentinel_addresses = [] 187 | for sentinel in self.sentinels: 188 | sentinel_addresses.append('%s:%s' % ( 189 | sentinel.connection_pool.connection_kwargs['host'], 190 | sentinel.connection_pool.connection_kwargs['port'], 191 | )) 192 | return '%s' % ( 193 | type(self).__name__, 194 | ','.join(sentinel_addresses)) 195 | 196 | def check_master_state(self, state, service_name): 197 | if not state['is_master'] or state['is_sdown'] or state['is_odown']: 198 | return False 199 | # Check if our sentinel doesn't see other nodes 200 | if state['num-other-sentinels'] < self.min_other_sentinels: 201 | return False 202 | return True 203 | 204 | def discover_master(self, service_name): 205 | """ 206 | Asks sentinel servers for the Redis master's address corresponding 207 | to the service labeled ``service_name``. 208 | 209 | Returns a pair (address, port) or raises MasterNotFoundError if no 210 | master is found. 211 | """ 212 | for sentinel_no, sentinel in enumerate(self.sentinels): 213 | try: 214 | masters = sentinel.sentinel_masters() 215 | except (ConnectionError, TimeoutError): 216 | continue 217 | state = masters.get(service_name) 218 | if state and self.check_master_state(state, service_name): 219 | # Put this sentinel at the top of the list 220 | self.sentinels[0], self.sentinels[sentinel_no] = ( 221 | sentinel, self.sentinels[0]) 222 | return state['ip'], state['port'] 223 | raise MasterNotFoundError("No master found for %r" % (service_name,)) 224 | 225 | def filter_slaves(self, slaves): 226 | "Remove slaves that are in an ODOWN or SDOWN state" 227 | slaves_alive = [] 228 | for slave in slaves: 229 | if slave['is_odown'] or slave['is_sdown']: 230 | continue 231 | slaves_alive.append((slave['ip'], slave['port'])) 232 | return slaves_alive 233 | 234 | def discover_slaves(self, service_name): 235 | "Returns a list of alive slaves for service ``service_name``" 236 | for sentinel in self.sentinels: 237 | try: 238 | slaves = sentinel.sentinel_slaves(service_name) 239 | except (ConnectionError, ResponseError, TimeoutError): 240 | continue 241 | slaves = self.filter_slaves(slaves) 242 | if slaves: 243 | return slaves 244 | return [] 245 | 246 | def master_for(self, service_name, redis_class=StrictRedis, 247 | connection_pool_class=SentinelConnectionPool, **kwargs): 248 | """ 249 | Returns a redis client instance for the ``service_name`` master. 250 | 251 | A SentinelConnectionPool class is used to retrive the master's 252 | address before establishing a new connection. 253 | 254 | NOTE: If the master's address has changed, any cached connections to 255 | the old master are closed. 256 | 257 | By default clients will be a redis.StrictRedis instance. Specify a 258 | different class to the ``redis_class`` argument if you desire 259 | something different. 260 | 261 | The ``connection_pool_class`` specifies the connection pool to use. 262 | The SentinelConnectionPool will be used by default. 263 | 264 | All other keyword arguments are merged with any connection_kwargs 265 | passed to this class and passed to the connection pool as keyword 266 | arguments to be used to initialize Redis connections. 267 | """ 268 | kwargs['is_master'] = True 269 | connection_kwargs = dict(self.connection_kwargs) 270 | connection_kwargs.update(kwargs) 271 | return redis_class(connection_pool=connection_pool_class( 272 | service_name, self, **connection_kwargs)) 273 | 274 | def slave_for(self, service_name, redis_class=StrictRedis, 275 | connection_pool_class=SentinelConnectionPool, **kwargs): 276 | """ 277 | Returns redis client instance for the ``service_name`` slave(s). 278 | 279 | A SentinelConnectionPool class is used to retrive the slave's 280 | address before establishing a new connection. 281 | 282 | By default clients will be a redis.StrictRedis instance. Specify a 283 | different class to the ``redis_class`` argument if you desire 284 | something different. 285 | 286 | The ``connection_pool_class`` specifies the connection pool to use. 287 | The SentinelConnectionPool will be used by default. 288 | 289 | All other keyword arguments are merged with any connection_kwargs 290 | passed to this class and passed to the connection pool as keyword 291 | arguments to be used to initialize Redis connections. 292 | """ 293 | kwargs['is_master'] = False 294 | connection_kwargs = dict(self.connection_kwargs) 295 | connection_kwargs.update(kwargs) 296 | return redis_class(connection_pool=connection_pool_class( 297 | service_name, self, **connection_kwargs)) 298 | -------------------------------------------------------------------------------- /redis/utils.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | 3 | 4 | try: 5 | import hiredis 6 | HIREDIS_AVAILABLE = True 7 | except ImportError: 8 | HIREDIS_AVAILABLE = False 9 | 10 | 11 | def from_url(url, db=None, **kwargs): 12 | """ 13 | Returns an active Redis client generated from the given database URL. 14 | 15 | Will attempt to extract the database id from the path url fragment, if 16 | none is provided. 17 | """ 18 | from redis.client import Redis 19 | return Redis.from_url(url, db, **kwargs) 20 | 21 | 22 | @contextmanager 23 | def pipeline(redis_obj): 24 | p = redis_obj.pipeline() 25 | yield p 26 | p.execute() 27 | 28 | 29 | class dummy(object): 30 | """ 31 | Instances of this class can be used as an attribute container. 32 | """ 33 | pass 34 | -------------------------------------------------------------------------------- /static/app.css: -------------------------------------------------------------------------------- 1 | @charset "utf-8"; 2 | 3 | @import url("https://fonts.googleapis.com/css?family=Roboto:300,400,500,700,900&subset=latin-ext"); 4 | @import url("https://fonts.googleapis.com/css?family=Inconsolata:400,700&subset=latin-ext"); 5 | 6 | * { 7 | box-sizing: border-box; 8 | outline: none; 9 | font-family: "Roboto", Arial, Helvetica, sans-serif; 10 | } 11 | 12 | body, 13 | html { 14 | margin: 0; 15 | padding: 17px 20px; 16 | font-family: "Roboto", Arial, Helvetica, sans-serif; 17 | font-size: 14px; 18 | } 19 | 20 | h1, h2, h3, h4, h5, h6 { 21 | margin: 0; 22 | padding: 0; 23 | } 24 | 25 | hr { 26 | border: 0; 27 | height: 5px; 28 | } 29 | 30 | input[type="search"] { 31 | font-size: 150%; 32 | padding: 15px 20px; 33 | width: 100%; 34 | border: 2px solid lightgrey; 35 | margin-bottom: 20px; 36 | /* safari fix */ 37 | -webkit-appearance: textfield; 38 | } 39 | 40 | .hide { 41 | display: none; 42 | } 43 | 44 | table { 45 | width: 100%; 46 | border-collapse: collapse; 47 | } 48 | 49 | table, th, td { 50 | border: 2px solid lightgrey; 51 | padding: 10px 15px; 52 | } 53 | 54 | table.no-border, table.no-border th, table.no-border td { 55 | border: 0; 56 | } 57 | 58 | table th { 59 | text-align: left; 60 | background: #eee; 61 | cursor: pointer; 62 | user-select: none; 63 | } 64 | 65 | button { 66 | border: 2px solid lightgrey; 67 | display: inline-block; 68 | background: lightgray; 69 | padding: 5px 10px; 70 | border-radius: 3px; 71 | cursor: pointer; 72 | font-weight: 700; 73 | font-size: 80%; 74 | line-height: 120%; 75 | vertical-align: middle; 76 | } 77 | 78 | button.big { 79 | padding: 8px 15px; 80 | font-size: 95%; 81 | } 82 | 83 | button.delete { 84 | border-color: lightcoral; 85 | background: lightcoral; 86 | color: white; 87 | } 88 | 89 | .grid-half { 90 | display: grid; 91 | grid-template-columns: 1fr 1fr; 92 | } 93 | 94 | .right { 95 | text-align: right; 96 | } 97 | 98 | .match-content { 99 | width: 1%; 100 | white-space: nowrap; 101 | } 102 | 103 | .red { 104 | color: lightcoral; 105 | } 106 | 107 | section#ops { 108 | margin-bottom: 20px; 109 | } 110 | 111 | div.column { 112 | display: inline-block; 113 | width: 100px; 114 | } 115 | 116 | section#ops label { 117 | display: inline-block; 118 | vertical-align: middle; 119 | margin-left: 20px; 120 | color: #bbb; 121 | font-size: 80%; 122 | font-weight: 700; 123 | } 124 | 125 | span.small { 126 | font-size: 70%; 127 | font-weight: 700; 128 | color: #aaa; 129 | display: block; 130 | text-transform: uppercase; 131 | padding-bottom: 2px; 132 | } 133 | 134 | span.medium { 135 | font-weight: 700; 136 | font-size: 115%; 137 | color: #444; 138 | display: inline-block; 139 | width: 60px; 140 | } 141 | 142 | section#item-form { 143 | position: fixed; 144 | z-index: 10; 145 | top: 25px; 146 | left: 50%; 147 | width: 900px; 148 | bottom: 25px; 149 | margin-left: -450px; 150 | border: 2px solid lightgrey; 151 | display: block; 152 | background: white; 153 | padding: 30px 20px; 154 | display: none; 155 | overflow: auto; 156 | } 157 | 158 | section#item-form input { 159 | width: 100%; 160 | border: 0; 161 | border: 1px dashed lightgray; 162 | padding: 8px 12px; 163 | font-size: 105%; 164 | } 165 | 166 | section#item-form > div { 167 | padding: 0 15px; 168 | } 169 | 170 | section#item-form td:first-child { 171 | width: 200px; 172 | } 173 | 174 | section#splashscreen { 175 | position: fixed; 176 | z-index: 9; 177 | left: 0; 178 | top: 0; 179 | right: 0; 180 | bottom: 0; 181 | background: rgba(0, 0, 0, 0.6); 182 | display: none; 183 | } 184 | 185 | section#exec-command { 186 | position: fixed; 187 | z-index: 10; 188 | top: 25px; 189 | left: 50%; 190 | bottom: 25px; 191 | width: 900px; 192 | margin-left: -450px; 193 | border: 2px solid lightgrey; 194 | display: none; 195 | background: white; 196 | overflow: auto; 197 | } 198 | 199 | section#exec-command input[type=search] { 200 | border: 0; 201 | border-bottom: 2px solid lightgrey; 202 | margin-bottom: 0; 203 | position: absolute; 204 | left: 0; 205 | top: 0; 206 | right: 0; 207 | } 208 | 209 | section#exec-command textarea { 210 | resize: none; 211 | border: none; 212 | position: absolute; 213 | left: 0; 214 | top: 65px; 215 | right: 0; 216 | bottom: 0; 217 | width: 100%; 218 | padding: 20px; 219 | font-family: 'Inconsolata', 'Courier New', Courier, monospace; 220 | font-size: 120%; 221 | } 222 | -------------------------------------------------------------------------------- /static/app.js: -------------------------------------------------------------------------------- 1 | const refreshRateForServerInfo = 5000; 2 | const queryKeySizeLimit = 500; 3 | const batchChunkSize = 50; 4 | 5 | let keyNum = 0; 6 | 7 | let queryInput; 8 | let splashScreen; 9 | let execCommand; 10 | let execCommandPopup; 11 | let execCommandQueryInput; 12 | let execCommandResults; 13 | 14 | let queryResults; 15 | let queryResultsTemplate; 16 | let serverInfo; 17 | let serverInfoTemplate; 18 | let itemFormInfo; 19 | let itemFormInfoTemplate; 20 | 21 | function showHelp() { 22 | let help = `hel* mathes hello 23 | h?llo matches hello, hallo and hxllo 24 | h*llo matches hllo and heeeello 25 | h[ae]llo matches hello and hallo, but not hillo 26 | h[^e]llo matches hallo, hbllo, ... but not hello 27 | h[a-b]llo matches hallo and hbllo`; 28 | alert(help); 29 | } 30 | 31 | function renderServerInfo() { 32 | fetch(path + "api/info").then(res => res.json()).then((payload) => { 33 | // if no keys db0 is not returned 34 | if (payload.hasOwnProperty("db" + db)) { 35 | payload.db = payload["db" + db]; 36 | } else { 37 | payload["db"] = { keys: 0 }; 38 | } 39 | keyNum = payload.db.keys; 40 | payload.total_commands_processed = numberFormatter(payload.total_commands_processed, 2); 41 | payload.db.keys = numberFormatter(payload.db.keys, 2); 42 | serverInfo.innerHTML = serverInfoTemplate(payload); 43 | }).catch((err) => { 44 | throw err; 45 | }); 46 | } 47 | 48 | function renderPopupForm(key, type) { 49 | fetch(path + "api/get?key=" + key + "&type=" + type).then(res => res.json()).then((payload) => { 50 | itemFormInfo.style.display = "block"; 51 | splashScreen.style.display = "block"; 52 | if (payload.type == "hash") payload.hash = true; 53 | itemFormInfo.innerHTML = itemFormInfoTemplate(payload); 54 | 55 | itemFormInfo.querySelector(".close").addEventListener("click", (evt) => { 56 | itemFormInfo.style.display = "none"; 57 | splashScreen.style.display = "none"; 58 | }, false); 59 | 60 | itemFormInfo.querySelectorAll(".delete").forEach((item, index) => { 61 | item.addEventListener("click", (evt) => { 62 | let row = evt.target.parentElement.parentElement; 63 | evt.target.parentElement.parentElement.parentElement.removeChild(row); 64 | }, false); 65 | }, false); 66 | 67 | try { 68 | itemFormInfo.querySelector(".add-attribute").addEventListener("click", (evt) => { 69 | let table = evt.target.parentElement.parentElement.querySelector("table tbody"); 70 | let row = table.insertRow(-1); 71 | let cell1 = row.insertCell(0); 72 | let cell2 = row.insertCell(1); 73 | let cell3 = row.insertCell(2); 74 | 75 | let keyInput = document.createElement("input"); 76 | keyInput.type = "text"; 77 | keyInput.name = "key"; 78 | 79 | let valueInput = document.createElement("input"); 80 | valueInput.type = "text"; 81 | valueInput.name = "value"; 82 | 83 | let deleteButton = document.createElement("button"); 84 | deleteButton.type = "button"; 85 | deleteButton.classList.add("delete"); 86 | deleteButton.innerText = "d"; 87 | deleteButton.addEventListener("click", (evt) => { 88 | let row = evt.target.parentElement.parentElement; 89 | evt.target.parentElement.parentElement.parentElement.removeChild(row); 90 | }, false); 91 | 92 | cell1.appendChild(keyInput); 93 | cell2.appendChild(valueInput); 94 | cell3.appendChild(deleteButton); 95 | 96 | }, false); 97 | } catch(err) {} 98 | 99 | itemFormInfo.querySelector(".save-item").addEventListener("click", (evt) => { 100 | let root = evt.target.parentElement.parentElement; 101 | let payload = {}; 102 | payload.key = key; 103 | payload.type = type; 104 | payload.value = []; 105 | 106 | if (type == "hash") { 107 | root.querySelectorAll("table tr").forEach((item, index) => { 108 | payload.value.push({ 109 | key: item.querySelector("input[name=key]").value, 110 | value: item.querySelector("input[name=value]").value, 111 | }); 112 | }, false); 113 | } else { 114 | payload.value = root.querySelector("input[name=value]").value 115 | } 116 | 117 | fetch(path + "api/set", { 118 | method: "POST", 119 | body: JSON.stringify(payload), 120 | }).then(res => res.json()).then((payload) => { 121 | if (payload.status) { 122 | itemFormInfo.style.display = "none"; 123 | splashScreen.style.display = "none"; 124 | } else { 125 | itemFormInfo.style.display = "none"; 126 | splashScreen.style.display = "none"; 127 | alert("Error saving item"); 128 | } 129 | }).catch((err) => { 130 | throw err; 131 | }); 132 | 133 | }, false); 134 | 135 | }).catch((err) => { 136 | throw err; 137 | }); 138 | } 139 | 140 | function renderSearchResults(payload) { 141 | queryResults.innerHTML = queryResultsTemplate(payload); 142 | 143 | // creates sortable tables based on class 144 | let sortableTables = queryResults.querySelectorAll(".sortable"); 145 | sortableTables.forEach(function (element) { 146 | tsorter.create(element); 147 | }); 148 | 149 | // auto select or deselect all rows 150 | queryResults.querySelector("thead input[type=checkbox]").addEventListener("click", (evt) => { 151 | if (evt.target.checked) { 152 | queryResults.querySelectorAll("tbody input[type=checkbox]").forEach((item, index) => { 153 | item.checked = true; 154 | }, false); 155 | } else { 156 | queryResults.querySelectorAll("tbody input[type=checkbox]").forEach((item, index) => { 157 | item.checked = false; 158 | }, false); 159 | } 160 | }, false); 161 | 162 | // row items ops 163 | queryResults.querySelectorAll("tbody tr").forEach((item, index) => { 164 | item.querySelector("button.delete").addEventListener("click", (evt) => { 165 | let itemKey = evt.target.parentElement.parentElement.dataset.key; 166 | let itemType = evt.target.parentElement.parentElement.dataset.type; 167 | let confirmDelete = confirm("Do you want to delete " + itemType + " item key with this id?\n" + itemKey); 168 | if (confirmDelete) { 169 | let sourceItem = evt.target.parentElement.parentElement; 170 | fetch(path + "api/del", { 171 | method: "POST", 172 | body: JSON.stringify({items: [itemKey]}), 173 | }).then(res => res.json()).then((payload) => { 174 | if (payload.status) { 175 | sourceItem.parentElement.removeChild(sourceItem); 176 | } 177 | }).catch((err) => { 178 | throw err; 179 | }); 180 | } 181 | }, false); 182 | 183 | // view data in popup 184 | item.querySelector("button.view").addEventListener("click", (evt) => { 185 | let itemKey = evt.target.parentElement.parentElement.dataset.key; 186 | let itemType = evt.target.parentElement.parentElement.dataset.type; 187 | renderPopupForm(itemKey, itemType); 188 | }, false); 189 | }); 190 | 191 | // bulk remove 192 | queryResults.querySelector("button.bulk-delete").addEventListener("click", (evt) => { 193 | if (queryResults.querySelectorAll("tbody input[type=checkbox]:checked").length > 0) { 194 | let confirmDelete = confirm("Do you SURE you want to delete selected items?"); 195 | if (confirmDelete) { 196 | let itemForDeletion = []; 197 | queryResults.querySelectorAll("tbody input[type=checkbox]").forEach((item, index) => { 198 | if (item.checked) { 199 | let sourceItem = item.parentElement.parentElement; 200 | itemForDeletion.push(sourceItem.dataset.key); 201 | } 202 | }, false); 203 | 204 | let i, j, chunkArray; 205 | for (i=0, j=itemForDeletion.length; i res.json()).then((payload) => { 211 | payload.items.forEach((item, index) => { 212 | let targetItem = queryResults.querySelector("tbody tr[data-key='" + item + "']"); 213 | targetItem.parentElement.removeChild(targetItem); 214 | }, false); 215 | }).catch((err) => { 216 | throw err; 217 | }); 218 | } 219 | } 220 | } else { 221 | alert("No items selected"); 222 | } 223 | }, false); 224 | } 225 | 226 | window.addEventListener("load", function (evt) { 227 | 228 | splashScreen = document.querySelector("section#splashscreen"); 229 | queryInput = document.querySelector("#query"); 230 | execCommand = document.querySelector("button.exec-cmd"); 231 | execCommandPopup = document.querySelector("#exec-command"); 232 | execCommandQueryInput = document.querySelector("#exec-command input"); 233 | execCommandResults = document.querySelector("#exec-command textarea"); 234 | 235 | queryResults = document.querySelector("section#query-results"); 236 | queryResultsTemplate = Handlebars.compile(document.querySelector("#query-results-tmpl").innerHTML); 237 | serverInfo = document.querySelector("#server-info"); 238 | serverInfoTemplate = Handlebars.compile(document.querySelector("#server-info-tmpl").innerHTML); 239 | itemFormInfo = document.querySelector("#item-form"); 240 | itemFormInfoTemplate = Handlebars.compile(document.querySelector("#item-form-tmpl").innerHTML); 241 | 242 | // perform query 243 | queryInput.addEventListener("keypress", function (evt) { 244 | var key = evt.which || evt.keyCode; 245 | if (key === 13 && this.value != "") { 246 | if ((this.value == "*") && (keyNum > queryKeySizeLimit)) { 247 | alert("Query not recomended. Dataset too large."); 248 | } else { 249 | this.disabled = true; 250 | fetch(path + "api/scan?q=" + this.value).then(res => res.json()).then((payload) => { 251 | renderSearchResults(payload); 252 | this.disabled = false; 253 | this.focus(); 254 | }).catch((err) => { 255 | throw err; 256 | }); 257 | } 258 | } 259 | }); 260 | 261 | // shows help 262 | document.querySelector("#ops .help").addEventListener("click", (evt) => { 263 | showHelp(); 264 | }, false); 265 | 266 | // shows execute command 267 | execCommand.addEventListener("click", (evt) => { 268 | execCommandPopup.style.display = "block"; 269 | splashScreen.style.display = "block"; 270 | execCommandQueryInput.focus(); 271 | 272 | 273 | }, false); 274 | 275 | // execute command 276 | execCommandQueryInput.addEventListener("keypress", function (evt) { 277 | var key = evt.which || evt.keyCode; 278 | if (key === 13 && this.value != "") { 279 | this.disabled = true; 280 | let dt = new Date(); 281 | execCommandResults.value += dt.toLocaleTimeString() + " >> " + this.value + "\n"; 282 | fetch(path + "api/exec", { 283 | method: "POST", 284 | body: JSON.stringify({cmd: this.value}), 285 | }).then(res => res.json()).then((payload) => { 286 | execCommandResults.value +=payload.message + "\n\n"; 287 | execCommandResults.scrollTop = execCommandResults.scrollHeight; 288 | this.disabled = false; 289 | this.focus(); 290 | }).catch((err) => { 291 | throw err; 292 | }); 293 | } 294 | }); 295 | 296 | // shows help 297 | document.querySelector("#ops .dump").addEventListener("click", (evt) => { 298 | alert("Not implemented yet!"); 299 | }, false); 300 | 301 | // on esc hides modal 302 | document.addEventListener("keydown", (evt) => { 303 | evt = evt || window.event; 304 | var isEscape = false; 305 | if ("key" in evt) { 306 | isEscape = (evt.key == "Escape" || evt.key == "Esc"); 307 | } else { 308 | isEscape = (evt.keyCode == 27); 309 | } 310 | if (isEscape) { 311 | itemFormInfo.style.display = "none"; 312 | splashScreen.style.display = "none"; 313 | execCommandPopup.style.display = "none"; 314 | } 315 | }, false); 316 | 317 | // on esc hides modal 318 | splashScreen.addEventListener("click", (evt) => { 319 | itemFormInfo.style.display = "none"; 320 | splashScreen.style.display = "none"; 321 | execCommandPopup.style.display = "none"; 322 | }, false); 323 | 324 | // refreshed info 325 | renderServerInfo(); 326 | this.setInterval(function() { 327 | renderServerInfo(); 328 | }, refreshRateForServerInfo); 329 | 330 | }, false); 331 | -------------------------------------------------------------------------------- /static/helpers.js: -------------------------------------------------------------------------------- 1 | /* author: https://stackoverflow.com/a/9462382 */ 2 | function numberFormatter(num, digits) { 3 | var si = [ 4 | { value: 1, symbol: "" }, 5 | { value: 1E3, symbol: "k" }, 6 | { value: 1E6, symbol: "M" }, 7 | { value: 1E9, symbol: "G" }, 8 | { value: 1E12, symbol: "T" }, 9 | { value: 1E15, symbol: "P" }, 10 | { value: 1E18, symbol: "E" } 11 | ]; 12 | var rx = /\.0+$|(\.[0-9]*[1-9])0+$/; 13 | var i; 14 | for (i = si.length - 1; i > 0; i--) { 15 | if (num >= si[i].value) { 16 | break; 17 | } 18 | } 19 | return (num / si[i].value).toFixed(digits).replace(rx, "$1") + si[i].symbol; 20 | } 21 | 22 | /* author: https://gist.github.com/TastyToast/5053642 */ 23 | Handlebars.registerHelper('truncate', function (str, len) { 24 | if (str.length > len && str.length > 0) { 25 | var new_str = str + " "; 26 | new_str = str.substr (0, len); 27 | new_str = str.substr (0, new_str.lastIndexOf(" ")); 28 | new_str = (new_str.length > 0) ? new_str : str.substr (0, len); 29 | 30 | return new Handlebars.SafeString ( new_str +'...' ); 31 | } 32 | return str; 33 | }); 34 | 35 | /*! 36 | * tsorter 2.0.0 - Copyright 2015 Terrill Dent, http://terrill.ca 37 | * JavaScript HTML Table Sorter 38 | * Released under MIT license, http://terrill.ca/sorting/tsorter/LICENSE 39 | */ 40 | 41 | var tsorter = (function () { 42 | 'use strict'; 43 | 44 | var sorterPrototype, 45 | addEvent, 46 | removeEvent, 47 | hasEventListener = !!document.addEventListener; 48 | 49 | if (!Object.create) { 50 | // Define Missing Function 51 | Object.create = function (prototype) { 52 | var Obj = function () { 53 | return undefined; 54 | }; 55 | Obj.prototype = prototype; 56 | return new Obj(); 57 | }; 58 | } 59 | 60 | // Cross Browser event binding 61 | addEvent = function (element, eventName, callback) { 62 | if (hasEventListener) { 63 | element.addEventListener(eventName, callback, false); 64 | } else { 65 | element.attachEvent('on' + eventName, callback); 66 | } 67 | }; 68 | 69 | // Cross Browser event removal 70 | removeEvent = function (element, eventName, callback) { 71 | if (hasEventListener) { 72 | element.removeEventListener(eventName, callback, false); 73 | } else { 74 | element.detachEvent('on' + eventName, callback); 75 | } 76 | }; 77 | 78 | sorterPrototype = { 79 | 80 | getCell: function (row) { 81 | var that = this; 82 | return that.trs[row].cells[that.column]; 83 | }, 84 | 85 | /* SORT 86 | * Sorts a particular column. If it has been sorted then call reverse 87 | * if not, then use quicksort to get it sorted. 88 | * Sets the arrow direction in the headers. 89 | * @param oTH - the table header cell () object that is clicked 90 | */ 91 | sort: function (e) { 92 | var that = this, 93 | th = e.target; 94 | 95 | // TODO: make sure target 'th' is not a child element of a 96 | // We can't use currentTarget because of backwards browser support 97 | // IE6,7,8 don't have it. 98 | 99 | // set the data retrieval function for this column 100 | that.column = th.cellIndex; 101 | that.get = that.getAccessor(th.getAttribute("data-tsorter")); 102 | 103 | if (th.classList.contains("ascend")) { 104 | th.classList.remove("ascend"); 105 | th.classList.add("descend"); 106 | that.reverseTable(); 107 | } else if (th.classList.contains("descend")) { 108 | th.classList.remove("descend"); 109 | th.classList.add("ascend"); 110 | that.reverseTable(); 111 | } else { 112 | for (var j = 0; j < that.ths.length; j++) { 113 | that.ths[j].classList.remove("ascend"); 114 | that.ths[j].classList.remove("descend"); 115 | } 116 | th.classList.add("ascend"); 117 | that.quicksort(0, that.trs.length); 118 | } 119 | }, 120 | 121 | /* 122 | * Choose Data Accessor Function 123 | * @param: the html structure type (from the data-type attribute) 124 | */ 125 | getAccessor: function (sortType) { 126 | var that = this, 127 | accessors = that.accessors; 128 | 129 | if (accessors && accessors[sortType]) { 130 | return accessors[sortType]; 131 | } 132 | 133 | switch (sortType) { 134 | case "link": 135 | return function (row) { 136 | return that.getCell(row).firstChild.firstChild.nodeValue; 137 | }; 138 | case "input": 139 | return function (row) { 140 | return that.getCell(row).firstChild.value; 141 | }; 142 | case "numeric": 143 | return function (row) { 144 | return parseFloat(that.getCell(row).firstChild.nodeValue, 10); 145 | }; 146 | default: 147 | /* Plain Text */ 148 | return function (row) { 149 | return that.getCell(row).firstChild.nodeValue; 150 | }; 151 | } 152 | }, 153 | 154 | /* Exchange 155 | * A complicated way of exchanging two rows in a table. 156 | * Exchanges rows at index i and j 157 | */ 158 | exchange: function (i, j) { 159 | var that = this, 160 | tbody = that.tbody, 161 | trs = that.trs, 162 | tmpNode; 163 | 164 | if (i === j + 1) { 165 | tbody.insertBefore(trs[i], trs[j]); 166 | } else if (j === i + 1) { 167 | tbody.insertBefore(trs[j], trs[i]); 168 | } else { 169 | tmpNode = tbody.replaceChild(trs[i], trs[j]); 170 | if (!trs[i]) { 171 | tbody.appendChild(tmpNode); 172 | } else { 173 | tbody.insertBefore(tmpNode, trs[i]); 174 | } 175 | } 176 | }, 177 | 178 | /* 179 | * REVERSE TABLE 180 | * Reverses a table ordering 181 | */ 182 | reverseTable: function () { 183 | var that = this, 184 | i; 185 | 186 | for (i = 1; i < that.trs.length; i++) { 187 | that.tbody.insertBefore(that.trs[i], that.trs[0]); 188 | } 189 | }, 190 | 191 | /* 192 | * QUICKSORT 193 | * @param: lo - the low index of the array to sort 194 | * @param: hi - the high index of the array to sort 195 | */ 196 | quicksort: function (lo, hi) { 197 | var i, j, pivot, 198 | that = this; 199 | 200 | if (hi <= lo + 1) { 201 | return; 202 | } 203 | 204 | if ((hi - lo) === 2) { 205 | if (that.get(hi - 1) > that.get(lo)) { 206 | that.exchange(hi - 1, lo); 207 | } 208 | return; 209 | } 210 | 211 | i = lo + 1; 212 | j = hi - 1; 213 | 214 | if (that.get(lo) > that.get(i)) { 215 | that.exchange(i, lo); 216 | } 217 | if (that.get(j) > that.get(lo)) { 218 | that.exchange(lo, j); 219 | } 220 | if (that.get(lo) > that.get(i)) { 221 | that.exchange(i, lo); 222 | } 223 | 224 | pivot = that.get(lo); 225 | 226 | while (true) { 227 | j--; 228 | while (pivot > that.get(j)) { 229 | j--; 230 | } 231 | i++; 232 | while (that.get(i) > pivot) { 233 | i++; 234 | } 235 | if (j <= i) { 236 | break; 237 | } 238 | that.exchange(i, j); 239 | } 240 | that.exchange(lo, j); 241 | 242 | if ((j - lo) < (hi - j)) { 243 | that.quicksort(lo, j); 244 | that.quicksort(j + 1, hi); 245 | } else { 246 | that.quicksort(j + 1, hi); 247 | that.quicksort(lo, j); 248 | } 249 | }, 250 | 251 | init: function (table, initialSortedColumn, customDataAccessors) { 252 | var that = this, 253 | i; 254 | 255 | if (typeof table === 'string') { 256 | table = document.getElementById(table); 257 | } 258 | 259 | that.table = table; 260 | that.ths = table.querySelectorAll("th.sort"); 261 | that.tbody = table.tBodies[0]; 262 | that.trs = that.tbody.getElementsByTagName("tr"); 263 | that.prevCol = (initialSortedColumn && initialSortedColumn > 0) ? initialSortedColumn : -1; 264 | that.accessors = customDataAccessors; 265 | that.boundSort = that.sort.bind(that); 266 | 267 | for (i = 0; i < that.ths.length; i++) { 268 | addEvent(that.ths[i], 'click', that.boundSort); 269 | } 270 | }, 271 | 272 | destroy: function () { 273 | var that = this, 274 | i; 275 | 276 | if (that.ths) { 277 | for (i = 0; i < that.ths.length; i++) { 278 | removeEvent(that.ths[i], 'click', that.boundSort); 279 | } 280 | } 281 | } 282 | }; 283 | 284 | // Create a new sorter given a table element 285 | return { 286 | create: function (table, initialSortedColumn, customDataAccessors) { 287 | var sorter = Object.create(sorterPrototype); 288 | sorter.init(table, initialSortedColumn, customDataAccessors); 289 | return sorter; 290 | } 291 | }; 292 | }()); 293 | -------------------------------------------------------------------------------- /static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Redis Marshal 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 | 20 | 21 | 22 | 23 | 24 |
25 |
26 |
27 | 28 | 29 | 30 |
31 | 32 |
33 | 34 |
35 | 36 | 37 |
38 | 39 |
40 | 41 | 42 | 43 | 76 | 77 | 91 | 92 | 127 | 128 | 129 | 130 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /waitress/__init__.py: -------------------------------------------------------------------------------- 1 | from waitress.server import create_server 2 | import logging 3 | 4 | def serve(app, **kw): 5 | _server = kw.pop('_server', create_server) # test shim 6 | _quiet = kw.pop('_quiet', False) # test shim 7 | _profile = kw.pop('_profile', False) # test shim 8 | if not _quiet: # pragma: no cover 9 | # idempotent if logging has already been set up 10 | logging.basicConfig() 11 | server = _server(app, **kw) 12 | if not _quiet: # pragma: no cover 13 | server.print_listen('Serving on http://{}:{}') 14 | if _profile: # pragma: no cover 15 | profile('server.run()', globals(), locals(), (), False) 16 | else: 17 | server.run() 18 | 19 | def serve_paste(app, global_conf, **kw): 20 | serve(app, **kw) 21 | return 0 22 | 23 | def profile(cmd, globals, locals, sort_order, callers): # pragma: no cover 24 | # runs a command under the profiler and print profiling output at shutdown 25 | import os 26 | import profile 27 | import pstats 28 | import tempfile 29 | fd, fn = tempfile.mkstemp() 30 | try: 31 | profile.runctx(cmd, globals, locals, fn) 32 | stats = pstats.Stats(fn) 33 | stats.strip_dirs() 34 | # calls,time,cumulative and cumulative,calls,time are useful 35 | stats.sort_stats(*(sort_order or ('cumulative', 'calls', 'time'))) 36 | if callers: 37 | stats.print_callers(.3) 38 | else: 39 | stats.print_stats(.3) 40 | finally: 41 | os.remove(fn) 42 | -------------------------------------------------------------------------------- /waitress/__main__.py: -------------------------------------------------------------------------------- 1 | from waitress.runner import run # pragma nocover 2 | run() # pragma nocover 3 | -------------------------------------------------------------------------------- /waitress/adjustments.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Adjustments are tunable parameters. 15 | """ 16 | import getopt 17 | import socket 18 | 19 | from waitress.compat import ( 20 | PY2, 21 | WIN, 22 | string_types, 23 | HAS_IPV6, 24 | ) 25 | 26 | truthy = frozenset(('t', 'true', 'y', 'yes', 'on', '1')) 27 | 28 | def asbool(s): 29 | """ Return the boolean value ``True`` if the case-lowered value of string 30 | input ``s`` is any of ``t``, ``true``, ``y``, ``on``, or ``1``, otherwise 31 | return the boolean value ``False``. If ``s`` is the value ``None``, 32 | return ``False``. If ``s`` is already one of the boolean values ``True`` 33 | or ``False``, return it.""" 34 | if s is None: 35 | return False 36 | if isinstance(s, bool): 37 | return s 38 | s = str(s).strip() 39 | return s.lower() in truthy 40 | 41 | def asoctal(s): 42 | """Convert the given octal string to an actual number.""" 43 | return int(s, 8) 44 | 45 | def aslist_cronly(value): 46 | if isinstance(value, string_types): 47 | value = filter(None, [x.strip() for x in value.splitlines()]) 48 | return list(value) 49 | 50 | def aslist(value): 51 | """ Return a list of strings, separating the input based on newlines 52 | and, if flatten=True (the default), also split on spaces within 53 | each line.""" 54 | values = aslist_cronly(value) 55 | result = [] 56 | for value in values: 57 | subvalues = value.split() 58 | result.extend(subvalues) 59 | return result 60 | 61 | def slash_fixed_str(s): 62 | s = s.strip() 63 | if s: 64 | # always have a leading slash, replace any number of leading slashes 65 | # with a single slash, and strip any trailing slashes 66 | s = '/' + s.lstrip('/').rstrip('/') 67 | return s 68 | 69 | class _str_marker(str): 70 | pass 71 | 72 | class _int_marker(int): 73 | pass 74 | 75 | class Adjustments(object): 76 | """This class contains tunable parameters. 77 | """ 78 | 79 | _params = ( 80 | ('host', str), 81 | ('port', int), 82 | ('ipv4', asbool), 83 | ('ipv6', asbool), 84 | ('listen', aslist), 85 | ('threads', int), 86 | ('trusted_proxy', str), 87 | ('url_scheme', str), 88 | ('url_prefix', slash_fixed_str), 89 | ('backlog', int), 90 | ('recv_bytes', int), 91 | ('send_bytes', int), 92 | ('outbuf_overflow', int), 93 | ('inbuf_overflow', int), 94 | ('connection_limit', int), 95 | ('cleanup_interval', int), 96 | ('channel_timeout', int), 97 | ('log_socket_errors', asbool), 98 | ('max_request_header_size', int), 99 | ('max_request_body_size', int), 100 | ('expose_tracebacks', asbool), 101 | ('ident', str), 102 | ('asyncore_loop_timeout', int), 103 | ('asyncore_use_poll', asbool), 104 | ('unix_socket', str), 105 | ('unix_socket_perms', asoctal), 106 | ) 107 | 108 | _param_map = dict(_params) 109 | 110 | # hostname or IP address to listen on 111 | host = _str_marker('0.0.0.0') 112 | 113 | # TCP port to listen on 114 | port = _int_marker(8080) 115 | 116 | listen = ['{}:{}'.format(host, port)] 117 | 118 | # mumber of threads available for tasks 119 | threads = 4 120 | 121 | # Host allowed to overrid ``wsgi.url_scheme`` via header 122 | trusted_proxy = None 123 | 124 | # default ``wsgi.url_scheme`` value 125 | url_scheme = 'http' 126 | 127 | # default ``SCRIPT_NAME`` value, also helps reset ``PATH_INFO`` 128 | # when nonempty 129 | url_prefix = '' 130 | 131 | # server identity (sent in Server: header) 132 | ident = 'waitress' 133 | 134 | # backlog is the value waitress passes to pass to socket.listen() This is 135 | # the maximum number of incoming TCP connections that will wait in an OS 136 | # queue for an available channel. From listen(1): "If a connection 137 | # request arrives when the queue is full, the client may receive an error 138 | # with an indication of ECONNREFUSED or, if the underlying protocol 139 | # supports retransmission, the request may be ignored so that a later 140 | # reattempt at connection succeeds." 141 | backlog = 1024 142 | 143 | # recv_bytes is the argument to pass to socket.recv(). 144 | recv_bytes = 8192 145 | 146 | # send_bytes is the number of bytes to send to socket.send(). Multiples 147 | # of 9000 should avoid partly-filled packets, but don't set this larger 148 | # than the TCP write buffer size. In Linux, /proc/sys/net/ipv4/tcp_wmem 149 | # controls the minimum, default, and maximum sizes of TCP write buffers. 150 | send_bytes = 18000 151 | 152 | # A tempfile should be created if the pending output is larger than 153 | # outbuf_overflow, which is measured in bytes. The default is 1MB. This 154 | # is conservative. 155 | outbuf_overflow = 1048576 156 | 157 | # A tempfile should be created if the pending input is larger than 158 | # inbuf_overflow, which is measured in bytes. The default is 512K. This 159 | # is conservative. 160 | inbuf_overflow = 524288 161 | 162 | # Stop creating new channels if too many are already active (integer). 163 | # Each channel consumes at least one file descriptor, and, depending on 164 | # the input and output body sizes, potentially up to three. The default 165 | # is conservative, but you may need to increase the number of file 166 | # descriptors available to the Waitress process on most platforms in 167 | # order to safely change it (see ``ulimit -a`` "open files" setting). 168 | # Note that this doesn't control the maximum number of TCP connections 169 | # that can be waiting for processing; the ``backlog`` argument controls 170 | # that. 171 | connection_limit = 100 172 | 173 | # Minimum seconds between cleaning up inactive channels. 174 | cleanup_interval = 30 175 | 176 | # Maximum seconds to leave an inactive connection open. 177 | channel_timeout = 120 178 | 179 | # Boolean: turn off to not log premature client disconnects. 180 | log_socket_errors = True 181 | 182 | # maximum number of bytes of all request headers combined (256K default) 183 | max_request_header_size = 262144 184 | 185 | # maximum number of bytes in request body (1GB default) 186 | max_request_body_size = 1073741824 187 | 188 | # expose tracebacks of uncaught exceptions 189 | expose_tracebacks = False 190 | 191 | # Path to a Unix domain socket to use. 192 | unix_socket = None 193 | 194 | # Path to a Unix domain socket to use. 195 | unix_socket_perms = 0o600 196 | 197 | # The socket options to set on receiving a connection. It is a list of 198 | # (level, optname, value) tuples. TCP_NODELAY disables the Nagle 199 | # algorithm for writes (Waitress already buffers its writes). 200 | socket_options = [ 201 | (socket.SOL_TCP, socket.TCP_NODELAY, 1), 202 | ] 203 | 204 | # The asyncore.loop timeout value 205 | asyncore_loop_timeout = 1 206 | 207 | # The asyncore.loop flag to use poll() instead of the default select(). 208 | asyncore_use_poll = False 209 | 210 | # Enable IPv4 by default 211 | ipv4 = True 212 | 213 | # Enable IPv6 by default 214 | ipv6 = True 215 | 216 | def __init__(self, **kw): 217 | 218 | if 'listen' in kw and ('host' in kw or 'port' in kw): 219 | raise ValueError('host and or port may not be set if listen is set.') 220 | 221 | for k, v in kw.items(): 222 | if k not in self._param_map: 223 | raise ValueError('Unknown adjustment %r' % k) 224 | setattr(self, k, self._param_map[k](v)) 225 | 226 | if (not isinstance(self.host, _str_marker) or 227 | not isinstance(self.port, _int_marker)): 228 | self.listen = ['{}:{}'.format(self.host, self.port)] 229 | 230 | enabled_families = socket.AF_UNSPEC 231 | 232 | if not self.ipv4 and not HAS_IPV6: # pragma: no cover 233 | raise ValueError( 234 | 'IPv4 is disabled but IPv6 is not available. Cowardly refusing to start.' 235 | ) 236 | 237 | if self.ipv4 and not self.ipv6: 238 | enabled_families = socket.AF_INET 239 | 240 | if not self.ipv4 and self.ipv6 and HAS_IPV6: 241 | enabled_families = socket.AF_INET6 242 | 243 | wanted_sockets = [] 244 | hp_pairs = [] 245 | for i in self.listen: 246 | if ':' in i: 247 | (host, port) = i.rsplit(":", 1) 248 | 249 | # IPv6 we need to make sure that we didn't split on the address 250 | if ']' in port: # pragma: nocover 251 | (host, port) = (i, str(self.port)) 252 | else: 253 | (host, port) = (i, str(self.port)) 254 | 255 | if WIN and PY2: # pragma: no cover 256 | try: 257 | # Try turning the port into an integer 258 | port = int(port) 259 | except: 260 | raise ValueError( 261 | 'Windows does not support service names instead of port numbers' 262 | ) 263 | 264 | try: 265 | if '[' in host and ']' in host: # pragma: nocover 266 | host = host.strip('[').rstrip(']') 267 | 268 | if host == '*': 269 | host = None 270 | 271 | for s in socket.getaddrinfo( 272 | host, 273 | port, 274 | enabled_families, 275 | socket.SOCK_STREAM, 276 | socket.IPPROTO_TCP, 277 | socket.AI_PASSIVE 278 | ): 279 | (family, socktype, proto, _, sockaddr) = s 280 | 281 | # It seems that getaddrinfo() may sometimes happily return 282 | # the same result multiple times, this of course makes 283 | # bind() very unhappy... 284 | # 285 | # Split on %, and drop the zone-index from the host in the 286 | # sockaddr. Works around a bug in OS X whereby 287 | # getaddrinfo() returns the same link-local interface with 288 | # two different zone-indices (which makes no sense what so 289 | # ever...) yet treats them equally when we attempt to bind(). 290 | if ( 291 | sockaddr[1] == 0 or 292 | (sockaddr[0].split('%', 1)[0], sockaddr[1]) not in hp_pairs 293 | ): 294 | wanted_sockets.append((family, socktype, proto, sockaddr)) 295 | hp_pairs.append((sockaddr[0].split('%', 1)[0], sockaddr[1])) 296 | except: 297 | raise ValueError('Invalid host/port specified.') 298 | 299 | self.listen = wanted_sockets 300 | 301 | @classmethod 302 | def parse_args(cls, argv): 303 | """Pre-parse command line arguments for input into __init__. Note that 304 | this does not cast values into adjustment types, it just creates a 305 | dictionary suitable for passing into __init__, where __init__ does the 306 | casting. 307 | """ 308 | long_opts = ['help', 'call'] 309 | for opt, cast in cls._params: 310 | opt = opt.replace('_', '-') 311 | if cast is asbool: 312 | long_opts.append(opt) 313 | long_opts.append('no-' + opt) 314 | else: 315 | long_opts.append(opt + '=') 316 | 317 | kw = { 318 | 'help': False, 319 | 'call': False, 320 | } 321 | 322 | opts, args = getopt.getopt(argv, '', long_opts) 323 | for opt, value in opts: 324 | param = opt.lstrip('-').replace('-', '_') 325 | 326 | if param == 'listen': 327 | kw['listen'] = '{} {}'.format(kw.get('listen', ''), value) 328 | continue 329 | 330 | if param.startswith('no_'): 331 | param = param[3:] 332 | kw[param] = 'false' 333 | elif param in ('help', 'call'): 334 | kw[param] = True 335 | elif cls._param_map[param] is asbool: 336 | kw[param] = 'true' 337 | else: 338 | kw[param] = value 339 | 340 | return kw, args 341 | -------------------------------------------------------------------------------- /waitress/buffers.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001-2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Buffers 15 | """ 16 | from io import BytesIO 17 | 18 | # copy_bytes controls the size of temp. strings for shuffling data around. 19 | COPY_BYTES = 1 << 18 # 256K 20 | 21 | # The maximum number of bytes to buffer in a simple string. 22 | STRBUF_LIMIT = 8192 23 | 24 | class FileBasedBuffer(object): 25 | 26 | remain = 0 27 | 28 | def __init__(self, file, from_buffer=None): 29 | self.file = file 30 | if from_buffer is not None: 31 | from_file = from_buffer.getfile() 32 | read_pos = from_file.tell() 33 | from_file.seek(0) 34 | while True: 35 | data = from_file.read(COPY_BYTES) 36 | if not data: 37 | break 38 | file.write(data) 39 | self.remain = int(file.tell() - read_pos) 40 | from_file.seek(read_pos) 41 | file.seek(read_pos) 42 | 43 | def __len__(self): 44 | return self.remain 45 | 46 | def __nonzero__(self): 47 | return True 48 | 49 | __bool__ = __nonzero__ # py3 50 | 51 | def append(self, s): 52 | file = self.file 53 | read_pos = file.tell() 54 | file.seek(0, 2) 55 | file.write(s) 56 | file.seek(read_pos) 57 | self.remain = self.remain + len(s) 58 | 59 | def get(self, numbytes=-1, skip=False): 60 | file = self.file 61 | if not skip: 62 | read_pos = file.tell() 63 | if numbytes < 0: 64 | # Read all 65 | res = file.read() 66 | else: 67 | res = file.read(numbytes) 68 | if skip: 69 | self.remain -= len(res) 70 | else: 71 | file.seek(read_pos) 72 | return res 73 | 74 | def skip(self, numbytes, allow_prune=0): 75 | if self.remain < numbytes: 76 | raise ValueError("Can't skip %d bytes in buffer of %d bytes" % ( 77 | numbytes, self.remain) 78 | ) 79 | self.file.seek(numbytes, 1) 80 | self.remain = self.remain - numbytes 81 | 82 | def newfile(self): 83 | raise NotImplementedError() 84 | 85 | def prune(self): 86 | file = self.file 87 | if self.remain == 0: 88 | read_pos = file.tell() 89 | file.seek(0, 2) 90 | sz = file.tell() 91 | file.seek(read_pos) 92 | if sz == 0: 93 | # Nothing to prune. 94 | return 95 | nf = self.newfile() 96 | while True: 97 | data = file.read(COPY_BYTES) 98 | if not data: 99 | break 100 | nf.write(data) 101 | self.file = nf 102 | 103 | def getfile(self): 104 | return self.file 105 | 106 | def close(self): 107 | if hasattr(self.file, 'close'): 108 | self.file.close() 109 | self.remain = 0 110 | 111 | class TempfileBasedBuffer(FileBasedBuffer): 112 | 113 | def __init__(self, from_buffer=None): 114 | FileBasedBuffer.__init__(self, self.newfile(), from_buffer) 115 | 116 | def newfile(self): 117 | from tempfile import TemporaryFile 118 | return TemporaryFile('w+b') 119 | 120 | class BytesIOBasedBuffer(FileBasedBuffer): 121 | 122 | def __init__(self, from_buffer=None): 123 | if from_buffer is not None: 124 | FileBasedBuffer.__init__(self, BytesIO(), from_buffer) 125 | else: 126 | # Shortcut. :-) 127 | self.file = BytesIO() 128 | 129 | def newfile(self): 130 | return BytesIO() 131 | 132 | class ReadOnlyFileBasedBuffer(FileBasedBuffer): 133 | # used as wsgi.file_wrapper 134 | 135 | def __init__(self, file, block_size=32768): 136 | self.file = file 137 | self.block_size = block_size # for __iter__ 138 | 139 | def prepare(self, size=None): 140 | if hasattr(self.file, 'seek') and hasattr(self.file, 'tell'): 141 | start_pos = self.file.tell() 142 | self.file.seek(0, 2) 143 | end_pos = self.file.tell() 144 | self.file.seek(start_pos) 145 | fsize = end_pos - start_pos 146 | if size is None: 147 | self.remain = fsize 148 | else: 149 | self.remain = min(fsize, size) 150 | return self.remain 151 | 152 | def get(self, numbytes=-1, skip=False): 153 | # never read more than self.remain (it can be user-specified) 154 | if numbytes == -1 or numbytes > self.remain: 155 | numbytes = self.remain 156 | file = self.file 157 | if not skip: 158 | read_pos = file.tell() 159 | res = file.read(numbytes) 160 | if skip: 161 | self.remain -= len(res) 162 | else: 163 | file.seek(read_pos) 164 | return res 165 | 166 | def __iter__(self): # called by task if self.filelike has no seek/tell 167 | return self 168 | 169 | def next(self): 170 | val = self.file.read(self.block_size) 171 | if not val: 172 | raise StopIteration 173 | return val 174 | 175 | __next__ = next # py3 176 | 177 | def append(self, s): 178 | raise NotImplementedError 179 | 180 | class OverflowableBuffer(object): 181 | """ 182 | This buffer implementation has four stages: 183 | - No data 184 | - Bytes-based buffer 185 | - BytesIO-based buffer 186 | - Temporary file storage 187 | The first two stages are fastest for simple transfers. 188 | """ 189 | 190 | overflowed = False 191 | buf = None 192 | strbuf = b'' # Bytes-based buffer. 193 | 194 | def __init__(self, overflow): 195 | # overflow is the maximum to be stored in a StringIO buffer. 196 | self.overflow = overflow 197 | 198 | def __len__(self): 199 | buf = self.buf 200 | if buf is not None: 201 | # use buf.__len__ rather than len(buf) FBO of not getting 202 | # OverflowError on Python 2 203 | return buf.__len__() 204 | else: 205 | return self.strbuf.__len__() 206 | 207 | def __nonzero__(self): 208 | # use self.__len__ rather than len(self) FBO of not getting 209 | # OverflowError on Python 2 210 | return self.__len__() > 0 211 | 212 | __bool__ = __nonzero__ # py3 213 | 214 | def _create_buffer(self): 215 | strbuf = self.strbuf 216 | if len(strbuf) >= self.overflow: 217 | self._set_large_buffer() 218 | else: 219 | self._set_small_buffer() 220 | buf = self.buf 221 | if strbuf: 222 | buf.append(self.strbuf) 223 | self.strbuf = b'' 224 | return buf 225 | 226 | def _set_small_buffer(self): 227 | self.buf = BytesIOBasedBuffer(self.buf) 228 | self.overflowed = False 229 | 230 | def _set_large_buffer(self): 231 | self.buf = TempfileBasedBuffer(self.buf) 232 | self.overflowed = True 233 | 234 | def append(self, s): 235 | buf = self.buf 236 | if buf is None: 237 | strbuf = self.strbuf 238 | if len(strbuf) + len(s) < STRBUF_LIMIT: 239 | self.strbuf = strbuf + s 240 | return 241 | buf = self._create_buffer() 242 | buf.append(s) 243 | # use buf.__len__ rather than len(buf) FBO of not getting 244 | # OverflowError on Python 2 245 | sz = buf.__len__() 246 | if not self.overflowed: 247 | if sz >= self.overflow: 248 | self._set_large_buffer() 249 | 250 | def get(self, numbytes=-1, skip=False): 251 | buf = self.buf 252 | if buf is None: 253 | strbuf = self.strbuf 254 | if not skip: 255 | return strbuf 256 | buf = self._create_buffer() 257 | return buf.get(numbytes, skip) 258 | 259 | def skip(self, numbytes, allow_prune=False): 260 | buf = self.buf 261 | if buf is None: 262 | if allow_prune and numbytes == len(self.strbuf): 263 | # We could slice instead of converting to 264 | # a buffer, but that would eat up memory in 265 | # large transfers. 266 | self.strbuf = b'' 267 | return 268 | buf = self._create_buffer() 269 | buf.skip(numbytes, allow_prune) 270 | 271 | def prune(self): 272 | """ 273 | A potentially expensive operation that removes all data 274 | already retrieved from the buffer. 275 | """ 276 | buf = self.buf 277 | if buf is None: 278 | self.strbuf = b'' 279 | return 280 | buf.prune() 281 | if self.overflowed: 282 | # use buf.__len__ rather than len(buf) FBO of not getting 283 | # OverflowError on Python 2 284 | sz = buf.__len__() 285 | if sz < self.overflow: 286 | # Revert to a faster buffer. 287 | self._set_small_buffer() 288 | 289 | def getfile(self): 290 | buf = self.buf 291 | if buf is None: 292 | buf = self._create_buffer() 293 | return buf.getfile() 294 | 295 | def close(self): 296 | buf = self.buf 297 | if buf is not None: 298 | buf.close() 299 | -------------------------------------------------------------------------------- /waitress/compat.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import types 3 | import platform 4 | import warnings 5 | 6 | try: 7 | import urlparse 8 | except ImportError: # pragma: no cover 9 | from urllib import parse as urlparse 10 | 11 | # True if we are running on Python 3. 12 | PY2 = sys.version_info[0] == 2 13 | PY3 = sys.version_info[0] == 3 14 | 15 | # True if we are running on Windows 16 | WIN = platform.system() == 'Windows' 17 | 18 | if PY3: # pragma: no cover 19 | string_types = str, 20 | integer_types = int, 21 | class_types = type, 22 | text_type = str 23 | binary_type = bytes 24 | long = int 25 | else: 26 | string_types = basestring, 27 | integer_types = (int, long) 28 | class_types = (type, types.ClassType) 29 | text_type = unicode 30 | binary_type = str 31 | long = long 32 | 33 | if PY3: # pragma: no cover 34 | from urllib.parse import unquote_to_bytes 35 | def unquote_bytes_to_wsgi(bytestring): 36 | return unquote_to_bytes(bytestring).decode('latin-1') 37 | else: 38 | from urlparse import unquote as unquote_to_bytes 39 | def unquote_bytes_to_wsgi(bytestring): 40 | return unquote_to_bytes(bytestring) 41 | 42 | def text_(s, encoding='latin-1', errors='strict'): 43 | """ If ``s`` is an instance of ``binary_type``, return 44 | ``s.decode(encoding, errors)``, otherwise return ``s``""" 45 | if isinstance(s, binary_type): 46 | return s.decode(encoding, errors) 47 | return s # pragma: no cover 48 | 49 | if PY3: # pragma: no cover 50 | def tostr(s): 51 | if isinstance(s, text_type): 52 | s = s.encode('latin-1') 53 | return str(s, 'latin-1', 'strict') 54 | 55 | def tobytes(s): 56 | return bytes(s, 'latin-1') 57 | else: 58 | tostr = str 59 | 60 | def tobytes(s): 61 | return s 62 | 63 | try: 64 | from Queue import ( 65 | Queue, 66 | Empty, 67 | ) 68 | except ImportError: # pragma: no cover 69 | from queue import ( 70 | Queue, 71 | Empty, 72 | ) 73 | 74 | if PY3: # pragma: no cover 75 | import builtins 76 | exec_ = getattr(builtins, "exec") 77 | 78 | def reraise(tp, value, tb=None): 79 | if value is None: 80 | value = tp 81 | if value.__traceback__ is not tb: 82 | raise value.with_traceback(tb) 83 | raise value 84 | 85 | del builtins 86 | 87 | else: # pragma: no cover 88 | def exec_(code, globs=None, locs=None): 89 | """Execute code in a namespace.""" 90 | if globs is None: 91 | frame = sys._getframe(1) 92 | globs = frame.f_globals 93 | if locs is None: 94 | locs = frame.f_locals 95 | del frame 96 | elif locs is None: 97 | locs = globs 98 | exec("""exec code in globs, locs""") 99 | 100 | exec_("""def reraise(tp, value, tb=None): 101 | raise tp, value, tb 102 | """) 103 | 104 | try: 105 | from StringIO import StringIO as NativeIO 106 | except ImportError: # pragma: no cover 107 | from io import StringIO as NativeIO 108 | 109 | try: 110 | import httplib 111 | except ImportError: # pragma: no cover 112 | from http import client as httplib 113 | 114 | try: 115 | MAXINT = sys.maxint 116 | except AttributeError: # pragma: no cover 117 | MAXINT = sys.maxsize 118 | 119 | 120 | # Fix for issue reported in https://github.com/Pylons/waitress/issues/138, 121 | # Python on Windows may not define IPPROTO_IPV6 in socket. 122 | import socket 123 | 124 | HAS_IPV6 = socket.has_ipv6 125 | 126 | if hasattr(socket, 'IPPROTO_IPV6') and hasattr(socket, 'IPV6_V6ONLY'): 127 | IPPROTO_IPV6 = socket.IPPROTO_IPV6 128 | IPV6_V6ONLY = socket.IPV6_V6ONLY 129 | else: # pragma: no cover 130 | if WIN: 131 | IPPROTO_IPV6 = 41 132 | IPV6_V6ONLY = 27 133 | else: 134 | warnings.warn( 135 | 'OS does not support required IPv6 socket flags. This is requirement ' 136 | 'for Waitress. Please open an issue at https://github.com/Pylons/waitress. ' 137 | 'IPv6 support has been disabled.', 138 | RuntimeWarning 139 | ) 140 | HAS_IPV6 = False 141 | -------------------------------------------------------------------------------- /waitress/parser.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """HTTP Request Parser 15 | 16 | This server uses asyncore to accept connections and do initial 17 | processing but threads to do work. 18 | """ 19 | import re 20 | from io import BytesIO 21 | 22 | from waitress.compat import ( 23 | tostr, 24 | urlparse, 25 | unquote_bytes_to_wsgi, 26 | ) 27 | 28 | from waitress.buffers import OverflowableBuffer 29 | 30 | from waitress.receiver import ( 31 | FixedStreamReceiver, 32 | ChunkedReceiver, 33 | ) 34 | 35 | from waitress.utilities import ( 36 | find_double_newline, 37 | RequestEntityTooLarge, 38 | RequestHeaderFieldsTooLarge, 39 | BadRequest, 40 | ) 41 | 42 | class ParsingError(Exception): 43 | pass 44 | 45 | class HTTPRequestParser(object): 46 | """A structure that collects the HTTP request. 47 | 48 | Once the stream is completed, the instance is passed to 49 | a server task constructor. 50 | """ 51 | completed = False # Set once request is completed. 52 | empty = False # Set if no request was made. 53 | expect_continue = False # client sent "Expect: 100-continue" header 54 | headers_finished = False # True when headers have been read 55 | header_plus = b'' 56 | chunked = False 57 | content_length = 0 58 | header_bytes_received = 0 59 | body_bytes_received = 0 60 | body_rcv = None 61 | version = '1.0' 62 | error = None 63 | connection_close = False 64 | 65 | # Other attributes: first_line, header, headers, command, uri, version, 66 | # path, query, fragment 67 | 68 | def __init__(self, adj): 69 | """ 70 | adj is an Adjustments object. 71 | """ 72 | # headers is a mapping containing keys translated to uppercase 73 | # with dashes turned into underscores. 74 | self.headers = {} 75 | self.adj = adj 76 | 77 | def received(self, data): 78 | """ 79 | Receives the HTTP stream for one request. Returns the number of 80 | bytes consumed. Sets the completed flag once both the header and the 81 | body have been received. 82 | """ 83 | if self.completed: 84 | return 0 # Can't consume any more. 85 | datalen = len(data) 86 | br = self.body_rcv 87 | if br is None: 88 | # In header. 89 | s = self.header_plus + data 90 | index = find_double_newline(s) 91 | if index >= 0: 92 | # Header finished. 93 | header_plus = s[:index] 94 | consumed = len(data) - (len(s) - index) 95 | # Remove preceeding blank lines. 96 | header_plus = header_plus.lstrip() 97 | if not header_plus: 98 | self.empty = True 99 | self.completed = True 100 | else: 101 | try: 102 | self.parse_header(header_plus) 103 | except ParsingError as e: 104 | self.error = BadRequest(e.args[0]) 105 | self.completed = True 106 | else: 107 | if self.body_rcv is None: 108 | # no content-length header and not a t-e: chunked 109 | # request 110 | self.completed = True 111 | if self.content_length > 0: 112 | max_body = self.adj.max_request_body_size 113 | # we won't accept this request if the content-length 114 | # is too large 115 | if self.content_length >= max_body: 116 | self.error = RequestEntityTooLarge( 117 | 'exceeds max_body of %s' % max_body) 118 | self.completed = True 119 | self.headers_finished = True 120 | return consumed 121 | else: 122 | # Header not finished yet. 123 | self.header_bytes_received += datalen 124 | max_header = self.adj.max_request_header_size 125 | if self.header_bytes_received >= max_header: 126 | # malformed header, we need to construct some request 127 | # on our own. we disregard the incoming(?) requests HTTP 128 | # version and just use 1.0. IOW someone just sent garbage 129 | # over the wire 130 | self.parse_header(b'GET / HTTP/1.0\n') 131 | self.error = RequestHeaderFieldsTooLarge( 132 | 'exceeds max_header of %s' % max_header) 133 | self.completed = True 134 | self.header_plus = s 135 | return datalen 136 | else: 137 | # In body. 138 | consumed = br.received(data) 139 | self.body_bytes_received += consumed 140 | max_body = self.adj.max_request_body_size 141 | if self.body_bytes_received >= max_body: 142 | # this will only be raised during t-e: chunked requests 143 | self.error = RequestEntityTooLarge( 144 | 'exceeds max_body of %s' % max_body) 145 | self.completed = True 146 | elif br.error: 147 | # garbage in chunked encoding input probably 148 | self.error = br.error 149 | self.completed = True 150 | elif br.completed: 151 | # The request (with the body) is ready to use. 152 | self.completed = True 153 | if self.chunked: 154 | # We've converted the chunked transfer encoding request 155 | # body into a normal request body, so we know its content 156 | # length; set the header here. We already popped the 157 | # TRANSFER_ENCODING header in parse_header, so this will 158 | # appear to the client to be an entirely non-chunked HTTP 159 | # request with a valid content-length. 160 | self.headers['CONTENT_LENGTH'] = str(br.__len__()) 161 | return consumed 162 | 163 | def parse_header(self, header_plus): 164 | """ 165 | Parses the header_plus block of text (the headers plus the 166 | first line of the request). 167 | """ 168 | index = header_plus.find(b'\n') 169 | if index >= 0: 170 | first_line = header_plus[:index].rstrip() 171 | header = header_plus[index + 1:] 172 | else: 173 | first_line = header_plus.rstrip() 174 | header = b'' 175 | 176 | self.first_line = first_line # for testing 177 | 178 | lines = get_header_lines(header) 179 | 180 | headers = self.headers 181 | for line in lines: 182 | index = line.find(b':') 183 | if index > 0: 184 | key = line[:index] 185 | if b'_' in key: 186 | continue 187 | value = line[index + 1:].strip() 188 | key1 = tostr(key.upper().replace(b'-', b'_')) 189 | # If a header already exists, we append subsequent values 190 | # seperated by a comma. Applications already need to handle 191 | # the comma seperated values, as HTTP front ends might do 192 | # the concatenation for you (behavior specified in RFC2616). 193 | try: 194 | headers[key1] += tostr(b', ' + value) 195 | except KeyError: 196 | headers[key1] = tostr(value) 197 | # else there's garbage in the headers? 198 | 199 | # command, uri, version will be bytes 200 | command, uri, version = crack_first_line(first_line) 201 | version = tostr(version) 202 | command = tostr(command) 203 | self.command = command 204 | self.version = version 205 | (self.proxy_scheme, 206 | self.proxy_netloc, 207 | self.path, 208 | self.query, self.fragment) = split_uri(uri) 209 | self.url_scheme = self.adj.url_scheme 210 | connection = headers.get('CONNECTION', '') 211 | 212 | if version == '1.0': 213 | if connection.lower() != 'keep-alive': 214 | self.connection_close = True 215 | 216 | if version == '1.1': 217 | # since the server buffers data from chunked transfers and clients 218 | # never need to deal with chunked requests, downstream clients 219 | # should not see the HTTP_TRANSFER_ENCODING header; we pop it 220 | # here 221 | te = headers.pop('TRANSFER_ENCODING', '') 222 | if te.lower() == 'chunked': 223 | self.chunked = True 224 | buf = OverflowableBuffer(self.adj.inbuf_overflow) 225 | self.body_rcv = ChunkedReceiver(buf) 226 | expect = headers.get('EXPECT', '').lower() 227 | self.expect_continue = expect == '100-continue' 228 | if connection.lower() == 'close': 229 | self.connection_close = True 230 | 231 | if not self.chunked: 232 | try: 233 | cl = int(headers.get('CONTENT_LENGTH', 0)) 234 | except ValueError: 235 | cl = 0 236 | self.content_length = cl 237 | if cl > 0: 238 | buf = OverflowableBuffer(self.adj.inbuf_overflow) 239 | self.body_rcv = FixedStreamReceiver(cl, buf) 240 | 241 | def get_body_stream(self): 242 | body_rcv = self.body_rcv 243 | if body_rcv is not None: 244 | return body_rcv.getfile() 245 | else: 246 | return BytesIO() 247 | 248 | def close(self): 249 | body_rcv = self.body_rcv 250 | if body_rcv is not None: 251 | body_rcv.getbuf().close() 252 | 253 | def split_uri(uri): 254 | # urlsplit handles byte input by returning bytes on py3, so 255 | # scheme, netloc, path, query, and fragment are bytes 256 | try: 257 | scheme, netloc, path, query, fragment = urlparse.urlsplit(uri) 258 | except UnicodeError: 259 | raise ParsingError('Bad URI') 260 | return ( 261 | tostr(scheme), 262 | tostr(netloc), 263 | unquote_bytes_to_wsgi(path), 264 | tostr(query), 265 | tostr(fragment), 266 | ) 267 | 268 | def get_header_lines(header): 269 | """ 270 | Splits the header into lines, putting multi-line headers together. 271 | """ 272 | r = [] 273 | lines = header.split(b'\n') 274 | for line in lines: 275 | if line.startswith((b' ', b'\t')): 276 | if not r: 277 | # http://corte.si/posts/code/pathod/pythonservers/index.html 278 | raise ParsingError('Malformed header line "%s"' % tostr(line)) 279 | r[-1] += line 280 | else: 281 | r.append(line) 282 | return r 283 | 284 | first_line_re = re.compile( 285 | b'([^ ]+) ' 286 | b'((?:[^ :?#]+://[^ ?#/]*(?:[0-9]{1,5})?)?[^ ]+)' 287 | b'(( HTTP/([0-9.]+))$|$)' 288 | ) 289 | 290 | def crack_first_line(line): 291 | m = first_line_re.match(line) 292 | if m is not None and m.end() == len(line): 293 | if m.group(3): 294 | version = m.group(5) 295 | else: 296 | version = None 297 | method = m.group(1) 298 | 299 | # the request methods that are currently defined are all uppercase: 300 | # https://www.iana.org/assignments/http-methods/http-methods.xhtml and 301 | # the request method is case sensitive according to 302 | # https://tools.ietf.org/html/rfc7231#section-4.1 303 | 304 | # By disallowing anything but uppercase methods we save poor 305 | # unsuspecting souls from sending lowercase HTTP methods to waitress 306 | # and having the request complete, while servers like nginx drop the 307 | # request onto the floor. 308 | if method != method.upper(): 309 | raise ParsingError('Malformed HTTP method "%s"' % tostr(method)) 310 | uri = m.group(2) 311 | return method, uri, version 312 | else: 313 | return b'', b'', b'' 314 | -------------------------------------------------------------------------------- /waitress/receiver.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Data Chunk Receiver 15 | """ 16 | 17 | from waitress.utilities import find_double_newline 18 | 19 | from waitress.utilities import BadRequest 20 | 21 | class FixedStreamReceiver(object): 22 | 23 | # See IStreamConsumer 24 | completed = False 25 | error = None 26 | 27 | def __init__(self, cl, buf): 28 | self.remain = cl 29 | self.buf = buf 30 | 31 | def __len__(self): 32 | return self.buf.__len__() 33 | 34 | def received(self, data): 35 | 'See IStreamConsumer' 36 | rm = self.remain 37 | if rm < 1: 38 | self.completed = True # Avoid any chance of spinning 39 | return 0 40 | datalen = len(data) 41 | if rm <= datalen: 42 | self.buf.append(data[:rm]) 43 | self.remain = 0 44 | self.completed = True 45 | return rm 46 | else: 47 | self.buf.append(data) 48 | self.remain -= datalen 49 | return datalen 50 | 51 | def getfile(self): 52 | return self.buf.getfile() 53 | 54 | def getbuf(self): 55 | return self.buf 56 | 57 | class ChunkedReceiver(object): 58 | 59 | chunk_remainder = 0 60 | control_line = b'' 61 | all_chunks_received = False 62 | trailer = b'' 63 | completed = False 64 | error = None 65 | 66 | # max_control_line = 1024 67 | # max_trailer = 65536 68 | 69 | def __init__(self, buf): 70 | self.buf = buf 71 | 72 | def __len__(self): 73 | return self.buf.__len__() 74 | 75 | def received(self, s): 76 | # Returns the number of bytes consumed. 77 | if self.completed: 78 | return 0 79 | orig_size = len(s) 80 | while s: 81 | rm = self.chunk_remainder 82 | if rm > 0: 83 | # Receive the remainder of a chunk. 84 | to_write = s[:rm] 85 | self.buf.append(to_write) 86 | written = len(to_write) 87 | s = s[written:] 88 | self.chunk_remainder -= written 89 | elif not self.all_chunks_received: 90 | # Receive a control line. 91 | s = self.control_line + s 92 | pos = s.find(b'\n') 93 | if pos < 0: 94 | # Control line not finished. 95 | self.control_line = s 96 | s = '' 97 | else: 98 | # Control line finished. 99 | line = s[:pos] 100 | s = s[pos + 1:] 101 | self.control_line = b'' 102 | line = line.strip() 103 | if line: 104 | # Begin a new chunk. 105 | semi = line.find(b';') 106 | if semi >= 0: 107 | # discard extension info. 108 | line = line[:semi] 109 | try: 110 | sz = int(line.strip(), 16) # hexadecimal 111 | except ValueError: # garbage in input 112 | self.error = BadRequest( 113 | 'garbage in chunked encoding input') 114 | sz = 0 115 | if sz > 0: 116 | # Start a new chunk. 117 | self.chunk_remainder = sz 118 | else: 119 | # Finished chunks. 120 | self.all_chunks_received = True 121 | # else expect a control line. 122 | else: 123 | # Receive the trailer. 124 | trailer = self.trailer + s 125 | if trailer.startswith(b'\r\n'): 126 | # No trailer. 127 | self.completed = True 128 | return orig_size - (len(trailer) - 2) 129 | elif trailer.startswith(b'\n'): 130 | # No trailer. 131 | self.completed = True 132 | return orig_size - (len(trailer) - 1) 133 | pos = find_double_newline(trailer) 134 | if pos < 0: 135 | # Trailer not finished. 136 | self.trailer = trailer 137 | s = b'' 138 | else: 139 | # Finished the trailer. 140 | self.completed = True 141 | self.trailer = trailer[:pos] 142 | return orig_size - (len(trailer) - pos) 143 | return orig_size 144 | 145 | def getfile(self): 146 | return self.buf.getfile() 147 | 148 | def getbuf(self): 149 | return self.buf 150 | -------------------------------------------------------------------------------- /waitress/runner.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2013 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Command line runner. 15 | """ 16 | 17 | from __future__ import print_function, unicode_literals 18 | 19 | import getopt 20 | import os 21 | import os.path 22 | import re 23 | import sys 24 | 25 | from waitress import serve 26 | from waitress.adjustments import Adjustments 27 | 28 | HELP = """\ 29 | Usage: 30 | 31 | {0} [OPTS] MODULE:OBJECT 32 | 33 | Standard options: 34 | 35 | --help 36 | Show this information. 37 | 38 | --call 39 | Call the given object to get the WSGI application. 40 | 41 | --host=ADDR 42 | Hostname or IP address on which to listen, default is '0.0.0.0', 43 | which means "all IP addresses on this host". 44 | 45 | Note: May not be used together with --listen 46 | 47 | --port=PORT 48 | TCP port on which to listen, default is '8080' 49 | 50 | Note: May not be used together with --listen 51 | 52 | --listen=ip:port 53 | Tell waitress to listen on an ip port combination. 54 | 55 | Example: 56 | 57 | --listen=127.0.0.1:8080 58 | --listen=[::1]:8080 59 | --listen=*:8080 60 | 61 | This option may be used multiple times to listen on multipe sockets. 62 | A wildcard for the hostname is also supported and will bind to both 63 | IPv4/IPv6 depending on whether they are enabled or disabled. 64 | 65 | --[no-]ipv4 66 | Toggle on/off IPv4 support. 67 | 68 | Example: 69 | 70 | --no-ipv4 71 | 72 | This will disable IPv4 socket support. This affects wildcard matching 73 | when generating the list of sockets. 74 | 75 | --[no-]ipv6 76 | Toggle on/off IPv6 support. 77 | 78 | Example: 79 | 80 | --no-ipv6 81 | 82 | This will turn on IPv6 socket support. This affects wildcard matching 83 | when generating a list of sockets. 84 | 85 | --unix-socket=PATH 86 | Path of Unix socket. If a socket path is specified, a Unix domain 87 | socket is made instead of the usual inet domain socket. 88 | 89 | Not available on Windows. 90 | 91 | --unix-socket-perms=PERMS 92 | Octal permissions to use for the Unix domain socket, default is 93 | '600'. 94 | 95 | --url-scheme=STR 96 | Default wsgi.url_scheme value, default is 'http'. 97 | 98 | --url-prefix=STR 99 | The ``SCRIPT_NAME`` WSGI environment value. Setting this to anything 100 | except the empty string will cause the WSGI ``SCRIPT_NAME`` value to be 101 | the value passed minus any trailing slashes you add, and it will cause 102 | the ``PATH_INFO`` of any request which is prefixed with this value to 103 | be stripped of the prefix. Default is the empty string. 104 | 105 | --ident=STR 106 | Server identity used in the 'Server' header in responses. Default 107 | is 'waitress'. 108 | 109 | Tuning options: 110 | 111 | --threads=INT 112 | Number of threads used to process application logic, default is 4. 113 | 114 | --backlog=INT 115 | Connection backlog for the server. Default is 1024. 116 | 117 | --recv-bytes=INT 118 | Number of bytes to request when calling socket.recv(). Default is 119 | 8192. 120 | 121 | --send-bytes=INT 122 | Number of bytes to send to socket.send(). Default is 18000. 123 | Multiples of 9000 should avoid partly-filled TCP packets. 124 | 125 | --outbuf-overflow=INT 126 | A temporary file should be created if the pending output is larger 127 | than this. Default is 1048576 (1MB). 128 | 129 | --inbuf-overflow=INT 130 | A temporary file should be created if the pending input is larger 131 | than this. Default is 524288 (512KB). 132 | 133 | --connection-limit=INT 134 | Stop creating new channelse if too many are already active. 135 | Default is 100. 136 | 137 | --cleanup-interval=INT 138 | Minimum seconds between cleaning up inactive channels. Default 139 | is 30. See '--channel-timeout'. 140 | 141 | --channel-timeout=INT 142 | Maximum number of seconds to leave inactive connections open. 143 | Default is 120. 'Inactive' is defined as 'has recieved no data 144 | from the client and has sent no data to the client'. 145 | 146 | --[no-]log-socket-errors 147 | Toggle whether premature client disconnect tracepacks ought to be 148 | logged. On by default. 149 | 150 | --max-request-header-size=INT 151 | Maximum size of all request headers combined. Default is 262144 152 | (256KB). 153 | 154 | --max-request-body-size=INT 155 | Maximum size of request body. Default is 1073741824 (1GB). 156 | 157 | --[no-]expose-tracebacks 158 | Toggle whether to expose tracebacks of unhandled exceptions to the 159 | client. Off by default. 160 | 161 | --asyncore-loop-timeout=INT 162 | The timeout value in seconds passed to asyncore.loop(). Default is 1. 163 | 164 | --asyncore-use-poll 165 | The use_poll argument passed to ``asyncore.loop()``. Helps overcome 166 | open file descriptors limit. Default is False. 167 | 168 | """ 169 | 170 | RUNNER_PATTERN = re.compile(r""" 171 | ^ 172 | (?P 173 | [a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)* 174 | ) 175 | : 176 | (?P 177 | [a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)* 178 | ) 179 | $ 180 | """, re.I | re.X) 181 | 182 | def match(obj_name): 183 | matches = RUNNER_PATTERN.match(obj_name) 184 | if not matches: 185 | raise ValueError("Malformed application '{0}'".format(obj_name)) 186 | return matches.group('module'), matches.group('object') 187 | 188 | def resolve(module_name, object_name): 189 | """Resolve a named object in a module.""" 190 | # We cast each segments due to an issue that has been found to manifest 191 | # in Python 2.6.6, but not 2.6.8, and may affect other revisions of Python 192 | # 2.6 and 2.7, whereby ``__import__`` chokes if the list passed in the 193 | # ``fromlist`` argument are unicode strings rather than 8-bit strings. 194 | # The error triggered is "TypeError: Item in ``fromlist '' not a string". 195 | # My guess is that this was fixed by checking against ``basestring`` 196 | # rather than ``str`` sometime between the release of 2.6.6 and 2.6.8, 197 | # but I've yet to go over the commits. I know, however, that the NEWS 198 | # file makes no mention of such a change to the behaviour of 199 | # ``__import__``. 200 | segments = [str(segment) for segment in object_name.split('.')] 201 | obj = __import__(module_name, fromlist=segments[:1]) 202 | for segment in segments: 203 | obj = getattr(obj, segment) 204 | return obj 205 | 206 | def show_help(stream, name, error=None): # pragma: no cover 207 | if error is not None: 208 | print('Error: {0}\n'.format(error), file=stream) 209 | print(HELP.format(name), file=stream) 210 | 211 | def show_exception(stream): 212 | exc_type, exc_value = sys.exc_info()[:2] 213 | args = getattr(exc_value, 'args', None) 214 | print( 215 | ( 216 | 'There was an exception ({0}) importing your module.\n' 217 | ).format( 218 | exc_type.__name__, 219 | ), 220 | file=stream 221 | ) 222 | if args: 223 | print('It had these arguments: ', file=stream) 224 | for idx, arg in enumerate(args, start=1): 225 | print('{0}. {1}\n'.format(idx, arg), file=stream) 226 | else: 227 | print('It had no arguments.', file=stream) 228 | 229 | def run(argv=sys.argv, _serve=serve): 230 | """Command line runner.""" 231 | name = os.path.basename(argv[0]) 232 | 233 | try: 234 | kw, args = Adjustments.parse_args(argv[1:]) 235 | except getopt.GetoptError as exc: 236 | show_help(sys.stderr, name, str(exc)) 237 | return 1 238 | 239 | if kw['help']: 240 | show_help(sys.stdout, name) 241 | return 0 242 | 243 | if len(args) != 1: 244 | show_help(sys.stderr, name, 'Specify one application only') 245 | return 1 246 | 247 | try: 248 | module, obj_name = match(args[0]) 249 | except ValueError as exc: 250 | show_help(sys.stderr, name, str(exc)) 251 | show_exception(sys.stderr) 252 | return 1 253 | 254 | # Add the current directory onto sys.path 255 | sys.path.append(os.getcwd()) 256 | 257 | # Get the WSGI function. 258 | try: 259 | app = resolve(module, obj_name) 260 | except ImportError: 261 | show_help(sys.stderr, name, "Bad module '{0}'".format(module)) 262 | show_exception(sys.stderr) 263 | return 1 264 | except AttributeError: 265 | show_help(sys.stderr, name, "Bad object name '{0}'".format(obj_name)) 266 | show_exception(sys.stderr) 267 | return 1 268 | if kw['call']: 269 | app = app() 270 | 271 | # These arguments are specific to the runner, not waitress itself. 272 | del kw['call'], kw['help'] 273 | 274 | _serve(app, **kw) 275 | return 0 276 | -------------------------------------------------------------------------------- /waitress/server.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import asyncore 16 | import os 17 | import os.path 18 | import socket 19 | import time 20 | 21 | from waitress import trigger 22 | from waitress.adjustments import Adjustments 23 | from waitress.channel import HTTPChannel 24 | from waitress.task import ThreadedTaskDispatcher 25 | from waitress.utilities import ( 26 | cleanup_unix_socket, 27 | logging_dispatcher, 28 | ) 29 | from waitress.compat import ( 30 | IPPROTO_IPV6, 31 | IPV6_V6ONLY, 32 | ) 33 | 34 | def create_server(application, 35 | map=None, 36 | _start=True, # test shim 37 | _sock=None, # test shim 38 | _dispatcher=None, # test shim 39 | **kw # adjustments 40 | ): 41 | """ 42 | if __name__ == '__main__': 43 | server = create_server(app) 44 | server.run() 45 | """ 46 | if application is None: 47 | raise ValueError( 48 | 'The "app" passed to ``create_server`` was ``None``. You forgot ' 49 | 'to return a WSGI app within your application.' 50 | ) 51 | adj = Adjustments(**kw) 52 | 53 | if map is None: # pragma: nocover 54 | map = {} 55 | 56 | dispatcher = _dispatcher 57 | if dispatcher is None: 58 | dispatcher = ThreadedTaskDispatcher() 59 | dispatcher.set_thread_count(adj.threads) 60 | 61 | if adj.unix_socket and hasattr(socket, 'AF_UNIX'): 62 | sockinfo = (socket.AF_UNIX, socket.SOCK_STREAM, None, None) 63 | return UnixWSGIServer( 64 | application, 65 | map, 66 | _start, 67 | _sock, 68 | dispatcher=dispatcher, 69 | adj=adj, 70 | sockinfo=sockinfo) 71 | 72 | effective_listen = [] 73 | last_serv = None 74 | for sockinfo in adj.listen: 75 | # When TcpWSGIServer is called, it registers itself in the map. This 76 | # side-effect is all we need it for, so we don't store a reference to 77 | # or return it to the user. 78 | last_serv = TcpWSGIServer( 79 | application, 80 | map, 81 | _start, 82 | _sock, 83 | dispatcher=dispatcher, 84 | adj=adj, 85 | sockinfo=sockinfo) 86 | effective_listen.append((last_serv.effective_host, last_serv.effective_port)) 87 | 88 | # We are running a single server, so we can just return the last server, 89 | # saves us from having to create one more object 90 | if len(adj.listen) == 1: 91 | # In this case we have no need to use a MultiSocketServer 92 | return last_serv 93 | 94 | # Return a class that has a utility function to print out the sockets it's 95 | # listening on, and has a .run() function. All of the TcpWSGIServers 96 | # registered themselves in the map above. 97 | return MultiSocketServer(map, adj, effective_listen, dispatcher) 98 | 99 | 100 | # This class is only ever used if we have multiple listen sockets. It allows 101 | # the serve() API to call .run() which starts the asyncore loop, and catches 102 | # SystemExit/KeyboardInterrupt so that it can atempt to cleanly shut down. 103 | class MultiSocketServer(object): 104 | asyncore = asyncore # test shim 105 | 106 | def __init__(self, 107 | map=None, 108 | adj=None, 109 | effective_listen=None, 110 | dispatcher=None, 111 | ): 112 | self.adj = adj 113 | self.map = map 114 | self.effective_listen = effective_listen 115 | self.task_dispatcher = dispatcher 116 | 117 | def print_listen(self, format_str): # pragma: nocover 118 | for l in self.effective_listen: 119 | l = list(l) 120 | 121 | if ':' in l[0]: 122 | l[0] = '[{}]'.format(l[0]) 123 | 124 | print(format_str.format(*l)) 125 | 126 | def run(self): 127 | try: 128 | self.asyncore.loop( 129 | timeout=self.adj.asyncore_loop_timeout, 130 | map=self.map, 131 | use_poll=self.adj.asyncore_use_poll, 132 | ) 133 | except (SystemExit, KeyboardInterrupt): 134 | self.task_dispatcher.shutdown() 135 | 136 | 137 | class BaseWSGIServer(logging_dispatcher, object): 138 | 139 | channel_class = HTTPChannel 140 | next_channel_cleanup = 0 141 | socketmod = socket # test shim 142 | asyncore = asyncore # test shim 143 | 144 | def __init__(self, 145 | application, 146 | map=None, 147 | _start=True, # test shim 148 | _sock=None, # test shim 149 | dispatcher=None, # dispatcher 150 | adj=None, # adjustments 151 | sockinfo=None, # opaque object 152 | **kw 153 | ): 154 | if adj is None: 155 | adj = Adjustments(**kw) 156 | if map is None: 157 | # use a nonglobal socket map by default to hopefully prevent 158 | # conflicts with apps and libs that use the asyncore global socket 159 | # map ala https://github.com/Pylons/waitress/issues/63 160 | map = {} 161 | if sockinfo is None: 162 | sockinfo = adj.listen[0] 163 | 164 | self.sockinfo = sockinfo 165 | self.family = sockinfo[0] 166 | self.socktype = sockinfo[1] 167 | self.application = application 168 | self.adj = adj 169 | self.trigger = trigger.trigger(map) 170 | if dispatcher is None: 171 | dispatcher = ThreadedTaskDispatcher() 172 | dispatcher.set_thread_count(self.adj.threads) 173 | 174 | self.task_dispatcher = dispatcher 175 | self.asyncore.dispatcher.__init__(self, _sock, map=map) 176 | if _sock is None: 177 | self.create_socket(self.family, self.socktype) 178 | if self.family == socket.AF_INET6: # pragma: nocover 179 | self.socket.setsockopt(IPPROTO_IPV6, IPV6_V6ONLY, 1) 180 | 181 | self.set_reuse_addr() 182 | self.bind_server_socket() 183 | self.effective_host, self.effective_port = self.getsockname() 184 | self.server_name = self.get_server_name(self.effective_host) 185 | self.active_channels = {} 186 | if _start: 187 | self.accept_connections() 188 | 189 | def bind_server_socket(self): 190 | raise NotImplementedError # pragma: no cover 191 | 192 | def get_server_name(self, ip): 193 | """Given an IP or hostname, try to determine the server name.""" 194 | if ip: 195 | server_name = str(ip) 196 | else: 197 | server_name = str(self.socketmod.gethostname()) 198 | 199 | # Convert to a host name if necessary. 200 | for c in server_name: 201 | if c != '.' and not c.isdigit(): 202 | return server_name 203 | try: 204 | if server_name == '0.0.0.0' or server_name == '::': 205 | return 'localhost' 206 | server_name = self.socketmod.gethostbyaddr(server_name)[0] 207 | except socket.error: # pragma: no cover 208 | pass 209 | return server_name 210 | 211 | def getsockname(self): 212 | raise NotImplementedError # pragma: no cover 213 | 214 | def accept_connections(self): 215 | self.accepting = True 216 | self.socket.listen(self.adj.backlog) # Get around asyncore NT limit 217 | 218 | def add_task(self, task): 219 | self.task_dispatcher.add_task(task) 220 | 221 | def readable(self): 222 | now = time.time() 223 | if now >= self.next_channel_cleanup: 224 | self.next_channel_cleanup = now + self.adj.cleanup_interval 225 | self.maintenance(now) 226 | return (self.accepting and len(self._map) < self.adj.connection_limit) 227 | 228 | def writable(self): 229 | return False 230 | 231 | def handle_read(self): 232 | pass 233 | 234 | def handle_connect(self): 235 | pass 236 | 237 | def handle_accept(self): 238 | try: 239 | v = self.accept() 240 | if v is None: 241 | return 242 | conn, addr = v 243 | except socket.error: 244 | # Linux: On rare occasions we get a bogus socket back from 245 | # accept. socketmodule.c:makesockaddr complains that the 246 | # address family is unknown. We don't want the whole server 247 | # to shut down because of this. 248 | if self.adj.log_socket_errors: 249 | self.logger.warning('server accept() threw an exception', 250 | exc_info=True) 251 | return 252 | self.set_socket_options(conn) 253 | addr = self.fix_addr(addr) 254 | self.channel_class(self, conn, addr, self.adj, map=self._map) 255 | 256 | def run(self): 257 | try: 258 | self.asyncore.loop( 259 | timeout=self.adj.asyncore_loop_timeout, 260 | map=self._map, 261 | use_poll=self.adj.asyncore_use_poll, 262 | ) 263 | except (SystemExit, KeyboardInterrupt): 264 | self.task_dispatcher.shutdown() 265 | 266 | def pull_trigger(self): 267 | self.trigger.pull_trigger() 268 | 269 | def set_socket_options(self, conn): 270 | pass 271 | 272 | def fix_addr(self, addr): 273 | return addr 274 | 275 | def maintenance(self, now): 276 | """ 277 | Closes channels that have not had any activity in a while. 278 | 279 | The timeout is configured through adj.channel_timeout (seconds). 280 | """ 281 | cutoff = now - self.adj.channel_timeout 282 | for channel in self.active_channels.values(): 283 | if (not channel.requests) and channel.last_activity < cutoff: 284 | channel.will_close = True 285 | 286 | def print_listen(self, format_str): # pragma: nocover 287 | print(format_str.format(self.effective_host, self.effective_port)) 288 | 289 | 290 | class TcpWSGIServer(BaseWSGIServer): 291 | 292 | def bind_server_socket(self): 293 | (_, _, _, sockaddr) = self.sockinfo 294 | self.bind(sockaddr) 295 | 296 | def getsockname(self): 297 | try: 298 | return self.socketmod.getnameinfo( 299 | self.socket.getsockname(), 300 | self.socketmod.NI_NUMERICSERV 301 | ) 302 | except: # pragma: no cover 303 | # This only happens on Linux because a DNS issue is considered a 304 | # temporary failure that will raise (even when NI_NAMEREQD is not 305 | # set). Instead we try again, but this time we just ask for the 306 | # numerichost and the numericserv (port) and return those. It is 307 | # better than nothing. 308 | return self.socketmod.getnameinfo( 309 | self.socket.getsockname(), 310 | self.socketmod.NI_NUMERICHOST | self.socketmod.NI_NUMERICSERV 311 | ) 312 | 313 | def set_socket_options(self, conn): 314 | for (level, optname, value) in self.adj.socket_options: 315 | conn.setsockopt(level, optname, value) 316 | 317 | 318 | if hasattr(socket, 'AF_UNIX'): 319 | 320 | class UnixWSGIServer(BaseWSGIServer): 321 | 322 | def __init__(self, 323 | application, 324 | map=None, 325 | _start=True, # test shim 326 | _sock=None, # test shim 327 | dispatcher=None, # dispatcher 328 | adj=None, # adjustments 329 | sockinfo=None, # opaque object 330 | **kw): 331 | if sockinfo is None: 332 | sockinfo = (socket.AF_UNIX, socket.SOCK_STREAM, None, None) 333 | 334 | super(UnixWSGIServer, self).__init__( 335 | application, 336 | map=map, 337 | _start=_start, 338 | _sock=_sock, 339 | dispatcher=dispatcher, 340 | adj=adj, 341 | sockinfo=sockinfo, 342 | **kw) 343 | 344 | def bind_server_socket(self): 345 | cleanup_unix_socket(self.adj.unix_socket) 346 | self.bind(self.adj.unix_socket) 347 | if os.path.exists(self.adj.unix_socket): 348 | os.chmod(self.adj.unix_socket, self.adj.unix_socket_perms) 349 | 350 | def getsockname(self): 351 | return ('unix', self.socket.getsockname()) 352 | 353 | def fix_addr(self, addr): 354 | return ('localhost', None) 355 | 356 | # Compatibility alias. 357 | WSGIServer = TcpWSGIServer 358 | -------------------------------------------------------------------------------- /waitress/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is necessary to make this directory a package. 3 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/__init__.py: -------------------------------------------------------------------------------- 1 | # package (for -m) 2 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/badcl.py: -------------------------------------------------------------------------------- 1 | def app(environ, start_response): # pragma: no cover 2 | body = b'abcdefghi' 3 | cl = len(body) 4 | if environ['PATH_INFO'] == '/short_body': 5 | cl = len(body) + 1 6 | if environ['PATH_INFO'] == '/long_body': 7 | cl = len(body) - 1 8 | start_response( 9 | '200 OK', 10 | [('Content-Length', str(cl)), ('Content-Type', 'text/plain')] 11 | ) 12 | return [body] 13 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/echo.py: -------------------------------------------------------------------------------- 1 | def app(environ, start_response): # pragma: no cover 2 | cl = environ.get('CONTENT_LENGTH', None) 3 | if cl is not None: 4 | cl = int(cl) 5 | body = environ['wsgi.input'].read(cl) 6 | cl = str(len(body)) 7 | start_response( 8 | '200 OK', 9 | [('Content-Length', cl), ('Content-Type', 'text/plain')] 10 | ) 11 | return [body] 12 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/error.py: -------------------------------------------------------------------------------- 1 | def app(environ, start_response): # pragma: no cover 2 | cl = environ.get('CONTENT_LENGTH', None) 3 | if cl is not None: 4 | cl = int(cl) 5 | body = environ['wsgi.input'].read(cl) 6 | cl = str(len(body)) 7 | if environ['PATH_INFO'] == '/before_start_response': 8 | raise ValueError('wrong') 9 | write = start_response( 10 | '200 OK', 11 | [('Content-Length', cl), ('Content-Type', 'text/plain')] 12 | ) 13 | if environ['PATH_INFO'] == '/after_write_cb': 14 | write('abc') 15 | if environ['PATH_INFO'] == '/in_generator': 16 | def foo(): 17 | yield 'abc' 18 | raise ValueError 19 | return foo() 20 | raise ValueError('wrong') 21 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/filewrapper.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | here = os.path.dirname(os.path.abspath(__file__)) 4 | fn = os.path.join(here, 'groundhog1.jpg') 5 | 6 | class KindaFilelike(object): # pragma: no cover 7 | 8 | def __init__(self, bytes): 9 | self.bytes = bytes 10 | 11 | def read(self, n): 12 | bytes = self.bytes[:n] 13 | self.bytes = self.bytes[n:] 14 | return bytes 15 | 16 | def app(environ, start_response): # pragma: no cover 17 | path_info = environ['PATH_INFO'] 18 | if path_info.startswith('/filelike'): 19 | f = open(fn, 'rb') 20 | f.seek(0, 2) 21 | cl = f.tell() 22 | f.seek(0) 23 | if path_info == '/filelike': 24 | headers = [ 25 | ('Content-Length', str(cl)), 26 | ('Content-Type', 'image/jpeg'), 27 | ] 28 | elif path_info == '/filelike_nocl': 29 | headers = [('Content-Type', 'image/jpeg')] 30 | elif path_info == '/filelike_shortcl': 31 | # short content length 32 | headers = [ 33 | ('Content-Length', '1'), 34 | ('Content-Type', 'image/jpeg'), 35 | ] 36 | else: 37 | # long content length (/filelike_longcl) 38 | headers = [ 39 | ('Content-Length', str(cl + 10)), 40 | ('Content-Type', 'image/jpeg'), 41 | ] 42 | else: 43 | data = open(fn, 'rb').read() 44 | cl = len(data) 45 | f = KindaFilelike(data) 46 | if path_info == '/notfilelike': 47 | headers = [ 48 | ('Content-Length', str(len(data))), 49 | ('Content-Type', 'image/jpeg'), 50 | ] 51 | elif path_info == '/notfilelike_nocl': 52 | headers = [('Content-Type', 'image/jpeg')] 53 | elif path_info == '/notfilelike_shortcl': 54 | # short content length 55 | headers = [ 56 | ('Content-Length', '1'), 57 | ('Content-Type', 'image/jpeg'), 58 | ] 59 | else: 60 | # long content length (/notfilelike_longcl) 61 | headers = [ 62 | ('Content-Length', str(cl + 10)), 63 | ('Content-Type', 'image/jpeg'), 64 | ] 65 | 66 | start_response( 67 | '200 OK', 68 | headers 69 | ) 70 | return environ['wsgi.file_wrapper'](f, 8192) 71 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/getline.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if __name__ == '__main__': 4 | try: 5 | from urllib.request import urlopen, URLError 6 | except ImportError: 7 | from urllib2 import urlopen, URLError 8 | 9 | url = sys.argv[1] 10 | headers = {'Content-Type': 'text/plain; charset=utf-8'} 11 | try: 12 | resp = urlopen(url) 13 | line = resp.readline().decode('ascii') # py3 14 | except URLError: 15 | line = 'failed to read %s' % url 16 | sys.stdout.write(line) 17 | sys.stdout.flush() 18 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/groundhog1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mitjafelicijan/redis-marshal/57c730529e86f803fc489e4d52973fd37fa12d53/waitress/tests/fixtureapps/groundhog1.jpg -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/nocl.py: -------------------------------------------------------------------------------- 1 | def chunks(l, n): # pragma: no cover 2 | """ Yield successive n-sized chunks from l. 3 | """ 4 | for i in range(0, len(l), n): 5 | yield l[i:i + n] 6 | 7 | def gen(body): # pragma: no cover 8 | for chunk in chunks(body, 10): 9 | yield chunk 10 | 11 | def app(environ, start_response): # pragma: no cover 12 | cl = environ.get('CONTENT_LENGTH', None) 13 | if cl is not None: 14 | cl = int(cl) 15 | body = environ['wsgi.input'].read(cl) 16 | start_response( 17 | '200 OK', 18 | [('Content-Type', 'text/plain')] 19 | ) 20 | if environ['PATH_INFO'] == '/list': 21 | return [body] 22 | if environ['PATH_INFO'] == '/list_lentwo': 23 | return [body[0:1], body[1:]] 24 | return gen(body) 25 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/runner.py: -------------------------------------------------------------------------------- 1 | def app(): # pragma: no cover 2 | return None 3 | 4 | def returns_app(): # pragma: no cover 5 | return app 6 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/sleepy.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | def app(environ, start_response): # pragma: no cover 4 | if environ['PATH_INFO'] == '/sleepy': 5 | time.sleep(2) 6 | body = b'sleepy returned' 7 | else: 8 | body = b'notsleepy returned' 9 | cl = str(len(body)) 10 | start_response( 11 | '200 OK', 12 | [('Content-Length', cl), ('Content-Type', 'text/plain')] 13 | ) 14 | return [body] 15 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/toolarge.py: -------------------------------------------------------------------------------- 1 | def app(environ, start_response): # pragma: no cover 2 | body = b'abcdef' 3 | cl = len(body) 4 | start_response( 5 | '200 OK', 6 | [('Content-Length', str(cl)), ('Content-Type', 'text/plain')] 7 | ) 8 | return [body] 9 | -------------------------------------------------------------------------------- /waitress/tests/fixtureapps/writecb.py: -------------------------------------------------------------------------------- 1 | def app(environ, start_response): # pragma: no cover 2 | path_info = environ['PATH_INFO'] 3 | if path_info == '/no_content_length': 4 | headers = [] 5 | else: 6 | headers = [('Content-Length', '9')] 7 | write = start_response('200 OK', headers) 8 | if path_info == '/long_body': 9 | write(b'abcdefghij') 10 | elif path_info == '/short_body': 11 | write(b'abcdefgh') 12 | else: 13 | write(b'abcdefghi') 14 | return [] 15 | -------------------------------------------------------------------------------- /waitress/tests/test_adjustments.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import socket 3 | 4 | from waitress.compat import ( 5 | PY2, 6 | WIN, 7 | ) 8 | 9 | if sys.version_info[:2] == (2, 6): # pragma: no cover 10 | import unittest2 as unittest 11 | else: # pragma: no cover 12 | import unittest 13 | 14 | class Test_asbool(unittest.TestCase): 15 | 16 | def _callFUT(self, s): 17 | from waitress.adjustments import asbool 18 | return asbool(s) 19 | 20 | def test_s_is_None(self): 21 | result = self._callFUT(None) 22 | self.assertEqual(result, False) 23 | 24 | def test_s_is_True(self): 25 | result = self._callFUT(True) 26 | self.assertEqual(result, True) 27 | 28 | def test_s_is_False(self): 29 | result = self._callFUT(False) 30 | self.assertEqual(result, False) 31 | 32 | def test_s_is_true(self): 33 | result = self._callFUT('True') 34 | self.assertEqual(result, True) 35 | 36 | def test_s_is_false(self): 37 | result = self._callFUT('False') 38 | self.assertEqual(result, False) 39 | 40 | def test_s_is_yes(self): 41 | result = self._callFUT('yes') 42 | self.assertEqual(result, True) 43 | 44 | def test_s_is_on(self): 45 | result = self._callFUT('on') 46 | self.assertEqual(result, True) 47 | 48 | def test_s_is_1(self): 49 | result = self._callFUT(1) 50 | self.assertEqual(result, True) 51 | 52 | class TestAdjustments(unittest.TestCase): 53 | 54 | def _hasIPv6(self): # pragma: nocover 55 | if not socket.has_ipv6: 56 | return False 57 | 58 | try: 59 | socket.getaddrinfo( 60 | '::1', 61 | 0, 62 | socket.AF_UNSPEC, 63 | socket.SOCK_STREAM, 64 | socket.IPPROTO_TCP, 65 | socket.AI_PASSIVE | socket.AI_ADDRCONFIG 66 | ) 67 | 68 | return True 69 | except socket.gaierror as e: 70 | # Check to see what the error is 71 | if e.errno == socket.EAI_ADDRFAMILY: 72 | return False 73 | else: 74 | raise e 75 | 76 | def _makeOne(self, **kw): 77 | from waitress.adjustments import Adjustments 78 | return Adjustments(**kw) 79 | 80 | def test_goodvars(self): 81 | inst = self._makeOne( 82 | host='localhost', 83 | port='8080', 84 | threads='5', 85 | trusted_proxy='192.168.1.1', 86 | url_scheme='https', 87 | backlog='20', 88 | recv_bytes='200', 89 | send_bytes='300', 90 | outbuf_overflow='400', 91 | inbuf_overflow='500', 92 | connection_limit='1000', 93 | cleanup_interval='1100', 94 | channel_timeout='1200', 95 | log_socket_errors='true', 96 | max_request_header_size='1300', 97 | max_request_body_size='1400', 98 | expose_tracebacks='true', 99 | ident='abc', 100 | asyncore_loop_timeout='5', 101 | asyncore_use_poll=True, 102 | unix_socket='/tmp/waitress.sock', 103 | unix_socket_perms='777', 104 | url_prefix='///foo/', 105 | ipv4=True, 106 | ipv6=False, 107 | ) 108 | 109 | self.assertEqual(inst.host, 'localhost') 110 | self.assertEqual(inst.port, 8080) 111 | self.assertEqual(inst.threads, 5) 112 | self.assertEqual(inst.trusted_proxy, '192.168.1.1') 113 | self.assertEqual(inst.url_scheme, 'https') 114 | self.assertEqual(inst.backlog, 20) 115 | self.assertEqual(inst.recv_bytes, 200) 116 | self.assertEqual(inst.send_bytes, 300) 117 | self.assertEqual(inst.outbuf_overflow, 400) 118 | self.assertEqual(inst.inbuf_overflow, 500) 119 | self.assertEqual(inst.connection_limit, 1000) 120 | self.assertEqual(inst.cleanup_interval, 1100) 121 | self.assertEqual(inst.channel_timeout, 1200) 122 | self.assertEqual(inst.log_socket_errors, True) 123 | self.assertEqual(inst.max_request_header_size, 1300) 124 | self.assertEqual(inst.max_request_body_size, 1400) 125 | self.assertEqual(inst.expose_tracebacks, True) 126 | self.assertEqual(inst.asyncore_loop_timeout, 5) 127 | self.assertEqual(inst.asyncore_use_poll, True) 128 | self.assertEqual(inst.ident, 'abc') 129 | self.assertEqual(inst.unix_socket, '/tmp/waitress.sock') 130 | self.assertEqual(inst.unix_socket_perms, 0o777) 131 | self.assertEqual(inst.url_prefix, '/foo') 132 | self.assertEqual(inst.ipv4, True) 133 | self.assertEqual(inst.ipv6, False) 134 | 135 | bind_pairs = [ 136 | sockaddr[:2] 137 | for (family, _, _, sockaddr) in inst.listen 138 | if family == socket.AF_INET 139 | ] 140 | 141 | # On Travis, somehow we start listening to two sockets when resolving 142 | # localhost... 143 | self.assertEqual(('127.0.0.1', 8080), bind_pairs[0]) 144 | 145 | def test_goodvar_listen(self): 146 | inst = self._makeOne(listen='127.0.0.1') 147 | 148 | bind_pairs = [(host, port) for (_, _, _, (host, port)) in inst.listen] 149 | 150 | self.assertEqual(bind_pairs, [('127.0.0.1', 8080)]) 151 | 152 | def test_default_listen(self): 153 | inst = self._makeOne() 154 | 155 | bind_pairs = [(host, port) for (_, _, _, (host, port)) in inst.listen] 156 | 157 | self.assertEqual(bind_pairs, [('0.0.0.0', 8080)]) 158 | 159 | def test_multiple_listen(self): 160 | inst = self._makeOne(listen='127.0.0.1:9090 127.0.0.1:8080') 161 | 162 | bind_pairs = [sockaddr[:2] for (_, _, _, sockaddr) in inst.listen] 163 | 164 | self.assertEqual(bind_pairs, 165 | [('127.0.0.1', 9090), 166 | ('127.0.0.1', 8080)]) 167 | 168 | def test_wildcard_listen(self): 169 | inst = self._makeOne(listen='*:8080') 170 | 171 | bind_pairs = [sockaddr[:2] for (_, _, _, sockaddr) in inst.listen] 172 | 173 | self.assertTrue(len(bind_pairs) >= 1) 174 | 175 | def test_ipv6_no_port(self): # pragma: nocover 176 | if not self._hasIPv6(): 177 | return 178 | 179 | inst = self._makeOne(listen='[::1]') 180 | 181 | bind_pairs = [sockaddr[:2] for (_, _, _, sockaddr) in inst.listen] 182 | 183 | self.assertEqual(bind_pairs, [('::1', 8080)]) 184 | 185 | def test_bad_port(self): 186 | self.assertRaises(ValueError, self._makeOne, listen='127.0.0.1:test') 187 | 188 | def test_service_port(self): 189 | if WIN and PY2: # pragma: no cover 190 | # On Windows and Python 2 this is broken, so we raise a ValueError 191 | self.assertRaises( 192 | ValueError, 193 | self._makeOne, 194 | listen='127.0.0.1:http', 195 | ) 196 | return 197 | 198 | inst = self._makeOne(listen='127.0.0.1:http 0.0.0.0:https') 199 | 200 | bind_pairs = [sockaddr[:2] for (_, _, _, sockaddr) in inst.listen] 201 | 202 | self.assertEqual(bind_pairs, [('127.0.0.1', 80), ('0.0.0.0', 443)]) 203 | 204 | def test_dont_mix_host_port_listen(self): 205 | self.assertRaises( 206 | ValueError, 207 | self._makeOne, 208 | host='localhost', 209 | port='8080', 210 | listen='127.0.0.1:8080', 211 | ) 212 | 213 | def test_badvar(self): 214 | self.assertRaises(ValueError, self._makeOne, nope=True) 215 | 216 | def test_ipv4_disabled(self): 217 | self.assertRaises(ValueError, self._makeOne, ipv4=False, listen="127.0.0.1:8080") 218 | 219 | def test_ipv6_disabled(self): 220 | self.assertRaises(ValueError, self._makeOne, ipv6=False, listen="[::]:8080") 221 | 222 | class TestCLI(unittest.TestCase): 223 | 224 | def parse(self, argv): 225 | from waitress.adjustments import Adjustments 226 | return Adjustments.parse_args(argv) 227 | 228 | def test_noargs(self): 229 | opts, args = self.parse([]) 230 | self.assertDictEqual(opts, {'call': False, 'help': False}) 231 | self.assertSequenceEqual(args, []) 232 | 233 | def test_help(self): 234 | opts, args = self.parse(['--help']) 235 | self.assertDictEqual(opts, {'call': False, 'help': True}) 236 | self.assertSequenceEqual(args, []) 237 | 238 | def test_call(self): 239 | opts, args = self.parse(['--call']) 240 | self.assertDictEqual(opts, {'call': True, 'help': False}) 241 | self.assertSequenceEqual(args, []) 242 | 243 | def test_both(self): 244 | opts, args = self.parse(['--call', '--help']) 245 | self.assertDictEqual(opts, {'call': True, 'help': True}) 246 | self.assertSequenceEqual(args, []) 247 | 248 | def test_positive_boolean(self): 249 | opts, args = self.parse(['--expose-tracebacks']) 250 | self.assertDictContainsSubset({'expose_tracebacks': 'true'}, opts) 251 | self.assertSequenceEqual(args, []) 252 | 253 | def test_negative_boolean(self): 254 | opts, args = self.parse(['--no-expose-tracebacks']) 255 | self.assertDictContainsSubset({'expose_tracebacks': 'false'}, opts) 256 | self.assertSequenceEqual(args, []) 257 | 258 | def test_cast_params(self): 259 | opts, args = self.parse([ 260 | '--host=localhost', 261 | '--port=80', 262 | '--unix-socket-perms=777' 263 | ]) 264 | self.assertDictContainsSubset({ 265 | 'host': 'localhost', 266 | 'port': '80', 267 | 'unix_socket_perms': '777', 268 | }, opts) 269 | self.assertSequenceEqual(args, []) 270 | 271 | def test_listen_params(self): 272 | opts, args = self.parse([ 273 | '--listen=test:80', 274 | ]) 275 | 276 | self.assertDictContainsSubset({ 277 | 'listen': ' test:80' 278 | }, opts) 279 | self.assertSequenceEqual(args, []) 280 | 281 | def test_multiple_listen_params(self): 282 | opts, args = self.parse([ 283 | '--listen=test:80', 284 | '--listen=test:8080', 285 | ]) 286 | 287 | self.assertDictContainsSubset({ 288 | 'listen': ' test:80 test:8080' 289 | }, opts) 290 | self.assertSequenceEqual(args, []) 291 | 292 | def test_bad_param(self): 293 | import getopt 294 | self.assertRaises(getopt.GetoptError, self.parse, ['--no-host']) 295 | -------------------------------------------------------------------------------- /waitress/tests/test_compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import unittest 4 | 5 | class Test_unquote_bytes_to_wsgi(unittest.TestCase): 6 | 7 | def _callFUT(self, v): 8 | from waitress.compat import unquote_bytes_to_wsgi 9 | return unquote_bytes_to_wsgi(v) 10 | 11 | def test_highorder(self): 12 | from waitress.compat import PY3 13 | val = b'/a%C5%9B' 14 | result = self._callFUT(val) 15 | if PY3: # pragma: no cover 16 | # PEP 3333 urlunquoted-latin1-decoded-bytes 17 | self.assertEqual(result, '/aÅ\x9b') 18 | else: # pragma: no cover 19 | # sanity 20 | self.assertEqual(result, b'/a\xc5\x9b') 21 | -------------------------------------------------------------------------------- /waitress/tests/test_init.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | class Test_serve(unittest.TestCase): 4 | 5 | def _callFUT(self, app, **kw): 6 | from waitress import serve 7 | return serve(app, **kw) 8 | 9 | def test_it(self): 10 | server = DummyServerFactory() 11 | app = object() 12 | result = self._callFUT(app, _server=server, _quiet=True) 13 | self.assertEqual(server.app, app) 14 | self.assertEqual(result, None) 15 | self.assertEqual(server.ran, True) 16 | 17 | class Test_serve_paste(unittest.TestCase): 18 | 19 | def _callFUT(self, app, **kw): 20 | from waitress import serve_paste 21 | return serve_paste(app, None, **kw) 22 | 23 | def test_it(self): 24 | server = DummyServerFactory() 25 | app = object() 26 | result = self._callFUT(app, _server=server, _quiet=True) 27 | self.assertEqual(server.app, app) 28 | self.assertEqual(result, 0) 29 | self.assertEqual(server.ran, True) 30 | 31 | class DummyServerFactory(object): 32 | ran = False 33 | 34 | def __call__(self, app, **kw): 35 | self.adj = DummyAdj(kw) 36 | self.app = app 37 | self.kw = kw 38 | return self 39 | 40 | def run(self): 41 | self.ran = True 42 | 43 | class DummyAdj(object): 44 | verbose = False 45 | 46 | def __init__(self, kw): 47 | self.__dict__.update(kw) 48 | -------------------------------------------------------------------------------- /waitress/tests/test_receiver.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | class TestFixedStreamReceiver(unittest.TestCase): 4 | 5 | def _makeOne(self, cl, buf): 6 | from waitress.receiver import FixedStreamReceiver 7 | return FixedStreamReceiver(cl, buf) 8 | 9 | def test_received_remain_lt_1(self): 10 | buf = DummyBuffer() 11 | inst = self._makeOne(0, buf) 12 | result = inst.received('a') 13 | self.assertEqual(result, 0) 14 | self.assertEqual(inst.completed, True) 15 | 16 | def test_received_remain_lte_datalen(self): 17 | buf = DummyBuffer() 18 | inst = self._makeOne(1, buf) 19 | result = inst.received('aa') 20 | self.assertEqual(result, 1) 21 | self.assertEqual(inst.completed, True) 22 | self.assertEqual(inst.completed, 1) 23 | self.assertEqual(inst.remain, 0) 24 | self.assertEqual(buf.data, ['a']) 25 | 26 | def test_received_remain_gt_datalen(self): 27 | buf = DummyBuffer() 28 | inst = self._makeOne(10, buf) 29 | result = inst.received('aa') 30 | self.assertEqual(result, 2) 31 | self.assertEqual(inst.completed, False) 32 | self.assertEqual(inst.remain, 8) 33 | self.assertEqual(buf.data, ['aa']) 34 | 35 | def test_getfile(self): 36 | buf = DummyBuffer() 37 | inst = self._makeOne(10, buf) 38 | self.assertEqual(inst.getfile(), buf) 39 | 40 | def test_getbuf(self): 41 | buf = DummyBuffer() 42 | inst = self._makeOne(10, buf) 43 | self.assertEqual(inst.getbuf(), buf) 44 | 45 | def test___len__(self): 46 | buf = DummyBuffer(['1', '2']) 47 | inst = self._makeOne(10, buf) 48 | self.assertEqual(inst.__len__(), 2) 49 | 50 | class TestChunkedReceiver(unittest.TestCase): 51 | 52 | def _makeOne(self, buf): 53 | from waitress.receiver import ChunkedReceiver 54 | return ChunkedReceiver(buf) 55 | 56 | def test_alreadycompleted(self): 57 | buf = DummyBuffer() 58 | inst = self._makeOne(buf) 59 | inst.completed = True 60 | result = inst.received(b'a') 61 | self.assertEqual(result, 0) 62 | self.assertEqual(inst.completed, True) 63 | 64 | def test_received_remain_gt_zero(self): 65 | buf = DummyBuffer() 66 | inst = self._makeOne(buf) 67 | inst.chunk_remainder = 100 68 | result = inst.received(b'a') 69 | self.assertEqual(inst.chunk_remainder, 99) 70 | self.assertEqual(result, 1) 71 | self.assertEqual(inst.completed, False) 72 | 73 | def test_received_control_line_notfinished(self): 74 | buf = DummyBuffer() 75 | inst = self._makeOne(buf) 76 | result = inst.received(b'a') 77 | self.assertEqual(inst.control_line, b'a') 78 | self.assertEqual(result, 1) 79 | self.assertEqual(inst.completed, False) 80 | 81 | def test_received_control_line_finished_garbage_in_input(self): 82 | buf = DummyBuffer() 83 | inst = self._makeOne(buf) 84 | result = inst.received(b'garbage\n') 85 | self.assertEqual(result, 8) 86 | self.assertTrue(inst.error) 87 | 88 | def test_received_control_line_finished_all_chunks_not_received(self): 89 | buf = DummyBuffer() 90 | inst = self._makeOne(buf) 91 | result = inst.received(b'a;discard\n') 92 | self.assertEqual(inst.control_line, b'') 93 | self.assertEqual(inst.chunk_remainder, 10) 94 | self.assertEqual(inst.all_chunks_received, False) 95 | self.assertEqual(result, 10) 96 | self.assertEqual(inst.completed, False) 97 | 98 | def test_received_control_line_finished_all_chunks_received(self): 99 | buf = DummyBuffer() 100 | inst = self._makeOne(buf) 101 | result = inst.received(b'0;discard\n') 102 | self.assertEqual(inst.control_line, b'') 103 | self.assertEqual(inst.all_chunks_received, True) 104 | self.assertEqual(result, 10) 105 | self.assertEqual(inst.completed, False) 106 | 107 | def test_received_trailer_startswith_crlf(self): 108 | buf = DummyBuffer() 109 | inst = self._makeOne(buf) 110 | inst.all_chunks_received = True 111 | result = inst.received(b'\r\n') 112 | self.assertEqual(result, 2) 113 | self.assertEqual(inst.completed, True) 114 | 115 | def test_received_trailer_startswith_lf(self): 116 | buf = DummyBuffer() 117 | inst = self._makeOne(buf) 118 | inst.all_chunks_received = True 119 | result = inst.received(b'\n') 120 | self.assertEqual(result, 1) 121 | self.assertEqual(inst.completed, True) 122 | 123 | def test_received_trailer_not_finished(self): 124 | buf = DummyBuffer() 125 | inst = self._makeOne(buf) 126 | inst.all_chunks_received = True 127 | result = inst.received(b'a') 128 | self.assertEqual(result, 1) 129 | self.assertEqual(inst.completed, False) 130 | 131 | def test_received_trailer_finished(self): 132 | buf = DummyBuffer() 133 | inst = self._makeOne(buf) 134 | inst.all_chunks_received = True 135 | result = inst.received(b'abc\r\n\r\n') 136 | self.assertEqual(inst.trailer, b'abc\r\n\r\n') 137 | self.assertEqual(result, 7) 138 | self.assertEqual(inst.completed, True) 139 | 140 | def test_getfile(self): 141 | buf = DummyBuffer() 142 | inst = self._makeOne(buf) 143 | self.assertEqual(inst.getfile(), buf) 144 | 145 | def test_getbuf(self): 146 | buf = DummyBuffer() 147 | inst = self._makeOne(buf) 148 | self.assertEqual(inst.getbuf(), buf) 149 | 150 | def test___len__(self): 151 | buf = DummyBuffer(['1', '2']) 152 | inst = self._makeOne(buf) 153 | self.assertEqual(inst.__len__(), 2) 154 | 155 | class DummyBuffer(object): 156 | 157 | def __init__(self, data=None): 158 | if data is None: 159 | data = [] 160 | self.data = data 161 | 162 | def append(self, s): 163 | self.data.append(s) 164 | 165 | def getfile(self): 166 | return self 167 | 168 | def __len__(self): 169 | return len(self.data) 170 | -------------------------------------------------------------------------------- /waitress/tests/test_regression.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2005 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Tests for waitress.channel maintenance logic 15 | """ 16 | import doctest 17 | 18 | class FakeSocket: # pragma: no cover 19 | data = '' 20 | setblocking = lambda *_: None 21 | close = lambda *_: None 22 | 23 | def __init__(self, no): 24 | self.no = no 25 | 26 | def fileno(self): 27 | return self.no 28 | 29 | def getpeername(self): 30 | return ('localhost', self.no) 31 | 32 | def send(self, data): 33 | self.data += data 34 | return len(data) 35 | 36 | def recv(self, data): 37 | return 'data' 38 | 39 | def zombies_test(): 40 | """Regression test for HTTPChannel.maintenance method 41 | 42 | Bug: This method checks for channels that have been "inactive" for a 43 | configured time. The bug was that last_activity is set at creation time 44 | but never updated during async channel activity (reads and writes), so 45 | any channel older than the configured timeout will be closed when a new 46 | channel is created, regardless of activity. 47 | 48 | >>> import time 49 | >>> import waitress.adjustments 50 | >>> config = waitress.adjustments.Adjustments() 51 | 52 | >>> from waitress.server import HTTPServer 53 | >>> class TestServer(HTTPServer): 54 | ... def bind(self, (ip, port)): 55 | ... print "Listening on %s:%d" % (ip or '*', port) 56 | >>> sb = TestServer('127.0.0.1', 80, start=False, verbose=True) 57 | Listening on 127.0.0.1:80 58 | 59 | First we confirm the correct behavior, where a channel with no activity 60 | for the timeout duration gets closed. 61 | 62 | >>> from waitress.channel import HTTPChannel 63 | >>> socket = FakeSocket(42) 64 | >>> channel = HTTPChannel(sb, socket, ('localhost', 42)) 65 | 66 | >>> channel.connected 67 | True 68 | 69 | >>> channel.last_activity -= int(config.channel_timeout) + 1 70 | 71 | >>> channel.next_channel_cleanup[0] = channel.creation_time - int( 72 | ... config.cleanup_interval) - 1 73 | 74 | >>> socket2 = FakeSocket(7) 75 | >>> channel2 = HTTPChannel(sb, socket2, ('localhost', 7)) 76 | 77 | >>> channel.connected 78 | False 79 | 80 | Write Activity 81 | -------------- 82 | 83 | Now we make sure that if there is activity the channel doesn't get closed 84 | incorrectly. 85 | 86 | >>> channel2.connected 87 | True 88 | 89 | >>> channel2.last_activity -= int(config.channel_timeout) + 1 90 | 91 | >>> channel2.handle_write() 92 | 93 | >>> channel2.next_channel_cleanup[0] = channel2.creation_time - int( 94 | ... config.cleanup_interval) - 1 95 | 96 | >>> socket3 = FakeSocket(3) 97 | >>> channel3 = HTTPChannel(sb, socket3, ('localhost', 3)) 98 | 99 | >>> channel2.connected 100 | True 101 | 102 | Read Activity 103 | -------------- 104 | 105 | We should test to see that read activity will update a channel as well. 106 | 107 | >>> channel3.connected 108 | True 109 | 110 | >>> channel3.last_activity -= int(config.channel_timeout) + 1 111 | 112 | >>> import waitress.parser 113 | >>> channel3.parser_class = ( 114 | ... waitress.parser.HTTPRequestParser) 115 | >>> channel3.handle_read() 116 | 117 | >>> channel3.next_channel_cleanup[0] = channel3.creation_time - int( 118 | ... config.cleanup_interval) - 1 119 | 120 | >>> socket4 = FakeSocket(4) 121 | >>> channel4 = HTTPChannel(sb, socket4, ('localhost', 4)) 122 | 123 | >>> channel3.connected 124 | True 125 | 126 | Main loop window 127 | ---------------- 128 | 129 | There is also a corner case we'll do a shallow test for where a 130 | channel can be closed waiting for the main loop. 131 | 132 | >>> channel4.last_activity -= 1 133 | 134 | >>> last_active = channel4.last_activity 135 | 136 | >>> channel4.set_async() 137 | 138 | >>> channel4.last_activity != last_active 139 | True 140 | 141 | """ 142 | 143 | def test_suite(): 144 | return doctest.DocTestSuite() 145 | -------------------------------------------------------------------------------- /waitress/tests/test_runner.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import os 3 | import sys 4 | 5 | if sys.version_info[:2] == (2, 6): # pragma: no cover 6 | import unittest2 as unittest 7 | else: # pragma: no cover 8 | import unittest 9 | 10 | from waitress import runner 11 | 12 | class Test_match(unittest.TestCase): 13 | 14 | def test_empty(self): 15 | self.assertRaisesRegexp( 16 | ValueError, "^Malformed application ''$", 17 | runner.match, '') 18 | 19 | def test_module_only(self): 20 | self.assertRaisesRegexp( 21 | ValueError, r"^Malformed application 'foo\.bar'$", 22 | runner.match, 'foo.bar') 23 | 24 | def test_bad_module(self): 25 | self.assertRaisesRegexp( 26 | ValueError, 27 | r"^Malformed application 'foo#bar:barney'$", 28 | runner.match, 'foo#bar:barney') 29 | 30 | def test_module_obj(self): 31 | self.assertTupleEqual( 32 | runner.match('foo.bar:fred.barney'), 33 | ('foo.bar', 'fred.barney')) 34 | 35 | class Test_resolve(unittest.TestCase): 36 | 37 | def test_bad_module(self): 38 | self.assertRaises( 39 | ImportError, 40 | runner.resolve, 'nonexistent', 'nonexistent_function') 41 | 42 | def test_nonexistent_function(self): 43 | self.assertRaisesRegexp( 44 | AttributeError, 45 | r"has no attribute 'nonexistent_function'", 46 | runner.resolve, 'os.path', 'nonexistent_function') 47 | 48 | def test_simple_happy_path(self): 49 | from os.path import exists 50 | self.assertIs(runner.resolve('os.path', 'exists'), exists) 51 | 52 | def test_complex_happy_path(self): 53 | # Ensure we can recursively resolve object attributes if necessary. 54 | self.assertEquals( 55 | runner.resolve('os.path', 'exists.__name__'), 56 | 'exists') 57 | 58 | class Test_run(unittest.TestCase): 59 | 60 | def match_output(self, argv, code, regex): 61 | argv = ['waitress-serve'] + argv 62 | with capture() as captured: 63 | self.assertEqual(runner.run(argv=argv), code) 64 | self.assertRegexpMatches(captured.getvalue(), regex) 65 | captured.close() 66 | 67 | def test_bad(self): 68 | self.match_output( 69 | ['--bad-opt'], 70 | 1, 71 | '^Error: option --bad-opt not recognized') 72 | 73 | def test_help(self): 74 | self.match_output( 75 | ['--help'], 76 | 0, 77 | "^Usage:\n\n waitress-serve") 78 | 79 | def test_no_app(self): 80 | self.match_output( 81 | [], 82 | 1, 83 | "^Error: Specify one application only") 84 | 85 | def test_multiple_apps_app(self): 86 | self.match_output( 87 | ['a:a', 'b:b'], 88 | 1, 89 | "^Error: Specify one application only") 90 | 91 | def test_bad_apps_app(self): 92 | self.match_output( 93 | ['a'], 94 | 1, 95 | "^Error: Malformed application 'a'") 96 | 97 | def test_bad_app_module(self): 98 | self.match_output( 99 | ['nonexistent:a'], 100 | 1, 101 | "^Error: Bad module 'nonexistent'") 102 | 103 | self.match_output( 104 | ['nonexistent:a'], 105 | 1, 106 | ( 107 | r"There was an exception \((ImportError|ModuleNotFoundError)\) " 108 | "importing your module.\n\nIt had these arguments: \n" 109 | "1. No module named '?nonexistent'?" 110 | ) 111 | ) 112 | 113 | def test_cwd_added_to_path(self): 114 | def null_serve(app, **kw): 115 | pass 116 | sys_path = sys.path 117 | current_dir = os.getcwd() 118 | try: 119 | os.chdir(os.path.dirname(__file__)) 120 | argv = [ 121 | 'waitress-serve', 122 | 'fixtureapps.runner:app', 123 | ] 124 | self.assertEqual(runner.run(argv=argv, _serve=null_serve), 0) 125 | finally: 126 | sys.path = sys_path 127 | os.chdir(current_dir) 128 | 129 | def test_bad_app_object(self): 130 | self.match_output( 131 | ['waitress.tests.fixtureapps.runner:a'], 132 | 1, 133 | "^Error: Bad object name 'a'") 134 | 135 | def test_simple_call(self): 136 | import waitress.tests.fixtureapps.runner as _apps 137 | def check_server(app, **kw): 138 | self.assertIs(app, _apps.app) 139 | self.assertDictEqual(kw, {'port': '80'}) 140 | argv = [ 141 | 'waitress-serve', 142 | '--port=80', 143 | 'waitress.tests.fixtureapps.runner:app', 144 | ] 145 | self.assertEqual(runner.run(argv=argv, _serve=check_server), 0) 146 | 147 | def test_returned_app(self): 148 | import waitress.tests.fixtureapps.runner as _apps 149 | def check_server(app, **kw): 150 | self.assertIs(app, _apps.app) 151 | self.assertDictEqual(kw, {'port': '80'}) 152 | argv = [ 153 | 'waitress-serve', 154 | '--port=80', 155 | '--call', 156 | 'waitress.tests.fixtureapps.runner:returns_app', 157 | ] 158 | self.assertEqual(runner.run(argv=argv, _serve=check_server), 0) 159 | 160 | class Test_helper(unittest.TestCase): 161 | 162 | def test_exception_logging(self): 163 | from waitress.runner import show_exception 164 | 165 | regex = ( 166 | r"There was an exception \(ImportError\) importing your module." 167 | r"\n\nIt had these arguments: \n1. My reason" 168 | ) 169 | 170 | with capture() as captured: 171 | try: 172 | raise ImportError("My reason") 173 | except ImportError: 174 | self.assertEqual(show_exception(sys.stderr), None) 175 | self.assertRegexpMatches( 176 | captured.getvalue(), 177 | regex 178 | ) 179 | captured.close() 180 | 181 | regex = ( 182 | r"There was an exception \(ImportError\) importing your module." 183 | r"\n\nIt had no arguments." 184 | ) 185 | 186 | with capture() as captured: 187 | try: 188 | raise ImportError 189 | except ImportError: 190 | self.assertEqual(show_exception(sys.stderr), None) 191 | self.assertRegexpMatches( 192 | captured.getvalue(), 193 | regex 194 | ) 195 | captured.close() 196 | 197 | @contextlib.contextmanager 198 | def capture(): 199 | from waitress.compat import NativeIO 200 | fd = NativeIO() 201 | sys.stdout = fd 202 | sys.stderr = fd 203 | yield fd 204 | sys.stdout = sys.__stdout__ 205 | sys.stderr = sys.__stderr__ 206 | -------------------------------------------------------------------------------- /waitress/tests/test_server.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import socket 3 | import unittest 4 | 5 | dummy_app = object() 6 | 7 | class TestWSGIServer(unittest.TestCase): 8 | 9 | def _makeOne(self, application=dummy_app, host='127.0.0.1', port=0, 10 | _dispatcher=None, adj=None, map=None, _start=True, 11 | _sock=None, _server=None): 12 | from waitress.server import create_server 13 | return create_server( 14 | application, 15 | host=host, 16 | port=port, 17 | map=map, 18 | _dispatcher=_dispatcher, 19 | _start=_start, 20 | _sock=_sock) 21 | 22 | def _makeOneWithMap(self, adj=None, _start=True, host='127.0.0.1', 23 | port=0, app=dummy_app): 24 | sock = DummySock() 25 | task_dispatcher = DummyTaskDispatcher() 26 | map = {} 27 | return self._makeOne( 28 | app, 29 | host=host, 30 | port=port, 31 | map=map, 32 | _sock=sock, 33 | _dispatcher=task_dispatcher, 34 | _start=_start, 35 | ) 36 | 37 | def _makeOneWithMulti(self, adj=None, _start=True, 38 | app=dummy_app, listen="127.0.0.1:0 127.0.0.1:0"): 39 | sock = DummySock() 40 | task_dispatcher = DummyTaskDispatcher() 41 | map = {} 42 | from waitress.server import create_server 43 | return create_server( 44 | app, 45 | listen=listen, 46 | map=map, 47 | _dispatcher=task_dispatcher, 48 | _start=_start, 49 | _sock=sock) 50 | 51 | def test_ctor_app_is_None(self): 52 | self.assertRaises(ValueError, self._makeOneWithMap, app=None) 53 | 54 | def test_ctor_start_true(self): 55 | inst = self._makeOneWithMap(_start=True) 56 | self.assertEqual(inst.accepting, True) 57 | self.assertEqual(inst.socket.listened, 1024) 58 | 59 | def test_ctor_makes_dispatcher(self): 60 | inst = self._makeOne(_start=False, map={}) 61 | self.assertEqual(inst.task_dispatcher.__class__.__name__, 62 | 'ThreadedTaskDispatcher') 63 | 64 | def test_ctor_start_false(self): 65 | inst = self._makeOneWithMap(_start=False) 66 | self.assertEqual(inst.accepting, False) 67 | 68 | def test_get_server_name_empty(self): 69 | inst = self._makeOneWithMap(_start=False) 70 | result = inst.get_server_name('') 71 | self.assertTrue(result) 72 | 73 | def test_get_server_name_with_ip(self): 74 | inst = self._makeOneWithMap(_start=False) 75 | result = inst.get_server_name('127.0.0.1') 76 | self.assertTrue(result) 77 | 78 | def test_get_server_name_with_hostname(self): 79 | inst = self._makeOneWithMap(_start=False) 80 | result = inst.get_server_name('fred.flintstone.com') 81 | self.assertEqual(result, 'fred.flintstone.com') 82 | 83 | def test_get_server_name_0000(self): 84 | inst = self._makeOneWithMap(_start=False) 85 | result = inst.get_server_name('0.0.0.0') 86 | self.assertEqual(result, 'localhost') 87 | 88 | def test_get_server_multi(self): 89 | inst = self._makeOneWithMulti() 90 | self.assertEqual(inst.__class__.__name__, 'MultiSocketServer') 91 | 92 | def test_run(self): 93 | inst = self._makeOneWithMap(_start=False) 94 | inst.asyncore = DummyAsyncore() 95 | inst.task_dispatcher = DummyTaskDispatcher() 96 | inst.run() 97 | self.assertTrue(inst.task_dispatcher.was_shutdown) 98 | 99 | def test_run_base_server(self): 100 | inst = self._makeOneWithMulti(_start=False) 101 | inst.asyncore = DummyAsyncore() 102 | inst.task_dispatcher = DummyTaskDispatcher() 103 | inst.run() 104 | self.assertTrue(inst.task_dispatcher.was_shutdown) 105 | 106 | def test_pull_trigger(self): 107 | inst = self._makeOneWithMap(_start=False) 108 | inst.trigger = DummyTrigger() 109 | inst.pull_trigger() 110 | self.assertEqual(inst.trigger.pulled, True) 111 | 112 | def test_add_task(self): 113 | task = DummyTask() 114 | inst = self._makeOneWithMap() 115 | inst.add_task(task) 116 | self.assertEqual(inst.task_dispatcher.tasks, [task]) 117 | self.assertFalse(task.serviced) 118 | 119 | def test_readable_not_accepting(self): 120 | inst = self._makeOneWithMap() 121 | inst.accepting = False 122 | self.assertFalse(inst.readable()) 123 | 124 | def test_readable_maplen_gt_connection_limit(self): 125 | inst = self._makeOneWithMap() 126 | inst.accepting = True 127 | inst.adj = DummyAdj 128 | inst._map = {'a': 1, 'b': 2} 129 | self.assertFalse(inst.readable()) 130 | 131 | def test_readable_maplen_lt_connection_limit(self): 132 | inst = self._makeOneWithMap() 133 | inst.accepting = True 134 | inst.adj = DummyAdj 135 | inst._map = {} 136 | self.assertTrue(inst.readable()) 137 | 138 | def test_readable_maintenance_false(self): 139 | import time 140 | inst = self._makeOneWithMap() 141 | then = time.time() + 1000 142 | inst.next_channel_cleanup = then 143 | L = [] 144 | inst.maintenance = lambda t: L.append(t) 145 | inst.readable() 146 | self.assertEqual(L, []) 147 | self.assertEqual(inst.next_channel_cleanup, then) 148 | 149 | def test_readable_maintenance_true(self): 150 | inst = self._makeOneWithMap() 151 | inst.next_channel_cleanup = 0 152 | L = [] 153 | inst.maintenance = lambda t: L.append(t) 154 | inst.readable() 155 | self.assertEqual(len(L), 1) 156 | self.assertNotEqual(inst.next_channel_cleanup, 0) 157 | 158 | def test_writable(self): 159 | inst = self._makeOneWithMap() 160 | self.assertFalse(inst.writable()) 161 | 162 | def test_handle_read(self): 163 | inst = self._makeOneWithMap() 164 | self.assertEqual(inst.handle_read(), None) 165 | 166 | def test_handle_connect(self): 167 | inst = self._makeOneWithMap() 168 | self.assertEqual(inst.handle_connect(), None) 169 | 170 | def test_handle_accept_wouldblock_socket_error(self): 171 | inst = self._makeOneWithMap() 172 | ewouldblock = socket.error(errno.EWOULDBLOCK) 173 | inst.socket = DummySock(toraise=ewouldblock) 174 | inst.handle_accept() 175 | self.assertEqual(inst.socket.accepted, False) 176 | 177 | def test_handle_accept_other_socket_error(self): 178 | inst = self._makeOneWithMap() 179 | eaborted = socket.error(errno.ECONNABORTED) 180 | inst.socket = DummySock(toraise=eaborted) 181 | inst.adj = DummyAdj 182 | def foo(): 183 | raise socket.error 184 | inst.accept = foo 185 | inst.logger = DummyLogger() 186 | inst.handle_accept() 187 | self.assertEqual(inst.socket.accepted, False) 188 | self.assertEqual(len(inst.logger.logged), 1) 189 | 190 | def test_handle_accept_noerror(self): 191 | inst = self._makeOneWithMap() 192 | innersock = DummySock() 193 | inst.socket = DummySock(acceptresult=(innersock, None)) 194 | inst.adj = DummyAdj 195 | L = [] 196 | inst.channel_class = lambda *arg, **kw: L.append(arg) 197 | inst.handle_accept() 198 | self.assertEqual(inst.socket.accepted, True) 199 | self.assertEqual(innersock.opts, [('level', 'optname', 'value')]) 200 | self.assertEqual(L, [(inst, innersock, None, inst.adj)]) 201 | 202 | def test_maintenance(self): 203 | inst = self._makeOneWithMap() 204 | 205 | class DummyChannel(object): 206 | requests = [] 207 | zombie = DummyChannel() 208 | zombie.last_activity = 0 209 | zombie.running_tasks = False 210 | inst.active_channels[100] = zombie 211 | inst.maintenance(10000) 212 | self.assertEqual(zombie.will_close, True) 213 | 214 | def test_backward_compatibility(self): 215 | from waitress.server import WSGIServer, TcpWSGIServer 216 | from waitress.adjustments import Adjustments 217 | self.assertTrue(WSGIServer is TcpWSGIServer) 218 | inst = WSGIServer(None, _start=False, port=1234) 219 | # Ensure the adjustment was actually applied. 220 | self.assertNotEqual(Adjustments.port, 1234) 221 | self.assertEqual(inst.adj.port, 1234) 222 | 223 | if hasattr(socket, 'AF_UNIX'): 224 | 225 | class TestUnixWSGIServer(unittest.TestCase): 226 | unix_socket = '/tmp/waitress.test.sock' 227 | 228 | def _makeOne(self, _start=True, _sock=None): 229 | from waitress.server import create_server 230 | return create_server( 231 | dummy_app, 232 | map={}, 233 | _start=_start, 234 | _sock=_sock, 235 | _dispatcher=DummyTaskDispatcher(), 236 | unix_socket=self.unix_socket, 237 | unix_socket_perms='600' 238 | ) 239 | 240 | def _makeDummy(self, *args, **kwargs): 241 | sock = DummySock(*args, **kwargs) 242 | sock.family = socket.AF_UNIX 243 | return sock 244 | 245 | def test_unix(self): 246 | inst = self._makeOne(_start=False) 247 | self.assertEqual(inst.socket.family, socket.AF_UNIX) 248 | self.assertEqual(inst.socket.getsockname(), self.unix_socket) 249 | 250 | def test_handle_accept(self): 251 | # Working on the assumption that we only have to test the happy path 252 | # for Unix domain sockets as the other paths should've been covered 253 | # by inet sockets. 254 | client = self._makeDummy() 255 | listen = self._makeDummy(acceptresult=(client, None)) 256 | inst = self._makeOne(_sock=listen) 257 | self.assertEqual(inst.accepting, True) 258 | self.assertEqual(inst.socket.listened, 1024) 259 | L = [] 260 | inst.channel_class = lambda *arg, **kw: L.append(arg) 261 | inst.handle_accept() 262 | self.assertEqual(inst.socket.accepted, True) 263 | self.assertEqual(client.opts, []) 264 | self.assertEqual( 265 | L, 266 | [(inst, client, ('localhost', None), inst.adj)] 267 | ) 268 | 269 | def test_creates_new_sockinfo(self): 270 | from waitress.server import UnixWSGIServer 271 | inst = UnixWSGIServer( 272 | dummy_app, 273 | unix_socket=self.unix_socket, 274 | unix_socket_perms='600' 275 | ) 276 | 277 | self.assertEqual(inst.sockinfo[0], socket.AF_UNIX) 278 | 279 | class DummySock(object): 280 | accepted = False 281 | blocking = False 282 | family = socket.AF_INET 283 | 284 | def __init__(self, toraise=None, acceptresult=(None, None)): 285 | self.toraise = toraise 286 | self.acceptresult = acceptresult 287 | self.bound = None 288 | self.opts = [] 289 | 290 | def bind(self, addr): 291 | self.bound = addr 292 | 293 | def accept(self): 294 | if self.toraise: 295 | raise self.toraise 296 | self.accepted = True 297 | return self.acceptresult 298 | 299 | def setblocking(self, x): 300 | self.blocking = True 301 | 302 | def fileno(self): 303 | return 10 304 | 305 | def getpeername(self): 306 | return '127.0.0.1' 307 | 308 | def setsockopt(self, *arg): 309 | self.opts.append(arg) 310 | 311 | def getsockopt(self, *arg): 312 | return 1 313 | 314 | def listen(self, num): 315 | self.listened = num 316 | 317 | def getsockname(self): 318 | return self.bound 319 | 320 | class DummyTaskDispatcher(object): 321 | 322 | def __init__(self): 323 | self.tasks = [] 324 | 325 | def add_task(self, task): 326 | self.tasks.append(task) 327 | 328 | def shutdown(self): 329 | self.was_shutdown = True 330 | 331 | class DummyTask(object): 332 | serviced = False 333 | start_response_called = False 334 | wrote_header = False 335 | status = '200 OK' 336 | 337 | def __init__(self): 338 | self.response_headers = {} 339 | self.written = '' 340 | 341 | def service(self): # pragma: no cover 342 | self.serviced = True 343 | 344 | class DummyAdj: 345 | connection_limit = 1 346 | log_socket_errors = True 347 | socket_options = [('level', 'optname', 'value')] 348 | cleanup_interval = 900 349 | channel_timeout = 300 350 | 351 | class DummyAsyncore(object): 352 | 353 | def loop(self, timeout=30.0, use_poll=False, map=None, count=None): 354 | raise SystemExit 355 | 356 | class DummyTrigger(object): 357 | 358 | def pull_trigger(self): 359 | self.pulled = True 360 | 361 | class DummyLogger(object): 362 | 363 | def __init__(self): 364 | self.logged = [] 365 | 366 | def warning(self, msg, **kw): 367 | self.logged.append(msg) 368 | -------------------------------------------------------------------------------- /waitress/tests/test_trigger.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import sys 4 | 5 | if not sys.platform.startswith("win"): 6 | 7 | class Test_trigger(unittest.TestCase): 8 | 9 | def _makeOne(self, map): 10 | from waitress.trigger import trigger 11 | return trigger(map) 12 | 13 | def test__close(self): 14 | map = {} 15 | inst = self._makeOne(map) 16 | fd = os.open(os.path.abspath(__file__), os.O_RDONLY) 17 | inst._fds = (fd,) 18 | inst.close() 19 | self.assertRaises(OSError, os.read, fd, 1) 20 | 21 | def test__physical_pull(self): 22 | map = {} 23 | inst = self._makeOne(map) 24 | inst._physical_pull() 25 | r = os.read(inst._fds[0], 1) 26 | self.assertEqual(r, b'x') 27 | 28 | def test_readable(self): 29 | map = {} 30 | inst = self._makeOne(map) 31 | self.assertEqual(inst.readable(), True) 32 | 33 | def test_writable(self): 34 | map = {} 35 | inst = self._makeOne(map) 36 | self.assertEqual(inst.writable(), False) 37 | 38 | def test_handle_connect(self): 39 | map = {} 40 | inst = self._makeOne(map) 41 | self.assertEqual(inst.handle_connect(), None) 42 | 43 | def test_close(self): 44 | map = {} 45 | inst = self._makeOne(map) 46 | self.assertEqual(inst.close(), None) 47 | self.assertEqual(inst._closed, True) 48 | 49 | def test_handle_close(self): 50 | map = {} 51 | inst = self._makeOne(map) 52 | self.assertEqual(inst.handle_close(), None) 53 | self.assertEqual(inst._closed, True) 54 | 55 | def test_pull_trigger_nothunk(self): 56 | map = {} 57 | inst = self._makeOne(map) 58 | self.assertEqual(inst.pull_trigger(), None) 59 | r = os.read(inst._fds[0], 1) 60 | self.assertEqual(r, b'x') 61 | 62 | def test_pull_trigger_thunk(self): 63 | map = {} 64 | inst = self._makeOne(map) 65 | self.assertEqual(inst.pull_trigger(True), None) 66 | self.assertEqual(len(inst.thunks), 1) 67 | r = os.read(inst._fds[0], 1) 68 | self.assertEqual(r, b'x') 69 | 70 | def test_handle_read_socket_error(self): 71 | map = {} 72 | inst = self._makeOne(map) 73 | result = inst.handle_read() 74 | self.assertEqual(result, None) 75 | 76 | def test_handle_read_no_socket_error(self): 77 | map = {} 78 | inst = self._makeOne(map) 79 | inst.pull_trigger() 80 | result = inst.handle_read() 81 | self.assertEqual(result, None) 82 | 83 | def test_handle_read_thunk(self): 84 | map = {} 85 | inst = self._makeOne(map) 86 | inst.pull_trigger() 87 | L = [] 88 | inst.thunks = [lambda: L.append(True)] 89 | result = inst.handle_read() 90 | self.assertEqual(result, None) 91 | self.assertEqual(L, [True]) 92 | self.assertEqual(inst.thunks, []) 93 | 94 | def test_handle_read_thunk_error(self): 95 | map = {} 96 | inst = self._makeOne(map) 97 | def errorthunk(): 98 | raise ValueError 99 | inst.pull_trigger(errorthunk) 100 | L = [] 101 | inst.log_info = lambda *arg: L.append(arg) 102 | result = inst.handle_read() 103 | self.assertEqual(result, None) 104 | self.assertEqual(len(L), 1) 105 | self.assertEqual(inst.thunks, []) 106 | -------------------------------------------------------------------------------- /waitress/tests/test_utilities.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import unittest 16 | 17 | class Test_parse_http_date(unittest.TestCase): 18 | 19 | def _callFUT(self, v): 20 | from waitress.utilities import parse_http_date 21 | return parse_http_date(v) 22 | 23 | def test_rfc850(self): 24 | val = 'Tuesday, 08-Feb-94 14:15:29 GMT' 25 | result = self._callFUT(val) 26 | self.assertEqual(result, 760716929) 27 | 28 | def test_rfc822(self): 29 | val = 'Sun, 08 Feb 1994 14:15:29 GMT' 30 | result = self._callFUT(val) 31 | self.assertEqual(result, 760716929) 32 | 33 | def test_neither(self): 34 | val = '' 35 | result = self._callFUT(val) 36 | self.assertEqual(result, 0) 37 | 38 | class Test_build_http_date(unittest.TestCase): 39 | 40 | def test_rountdrip(self): 41 | from waitress.utilities import build_http_date, parse_http_date 42 | from time import time 43 | t = int(time()) 44 | self.assertEqual(t, parse_http_date(build_http_date(t))) 45 | 46 | class Test_unpack_rfc850(unittest.TestCase): 47 | 48 | def _callFUT(self, val): 49 | from waitress.utilities import unpack_rfc850, rfc850_reg 50 | return unpack_rfc850(rfc850_reg.match(val.lower())) 51 | 52 | def test_it(self): 53 | val = 'Tuesday, 08-Feb-94 14:15:29 GMT' 54 | result = self._callFUT(val) 55 | self.assertEqual(result, (1994, 2, 8, 14, 15, 29, 0, 0, 0)) 56 | 57 | class Test_unpack_rfc_822(unittest.TestCase): 58 | 59 | def _callFUT(self, val): 60 | from waitress.utilities import unpack_rfc822, rfc822_reg 61 | return unpack_rfc822(rfc822_reg.match(val.lower())) 62 | 63 | def test_it(self): 64 | val = 'Sun, 08 Feb 1994 14:15:29 GMT' 65 | result = self._callFUT(val) 66 | self.assertEqual(result, (1994, 2, 8, 14, 15, 29, 0, 0, 0)) 67 | 68 | class Test_find_double_newline(unittest.TestCase): 69 | 70 | def _callFUT(self, val): 71 | from waitress.utilities import find_double_newline 72 | return find_double_newline(val) 73 | 74 | def test_empty(self): 75 | self.assertEqual(self._callFUT(b''), -1) 76 | 77 | def test_one_linefeed(self): 78 | self.assertEqual(self._callFUT(b'\n'), -1) 79 | 80 | def test_double_linefeed(self): 81 | self.assertEqual(self._callFUT(b'\n\n'), 2) 82 | 83 | def test_one_crlf(self): 84 | self.assertEqual(self._callFUT(b'\r\n'), -1) 85 | 86 | def test_double_crfl(self): 87 | self.assertEqual(self._callFUT(b'\r\n\r\n'), 4) 88 | 89 | def test_mixed(self): 90 | self.assertEqual(self._callFUT(b'\n\n00\r\n\r\n'), 2) 91 | 92 | class Test_logging_dispatcher(unittest.TestCase): 93 | 94 | def _makeOne(self): 95 | from waitress.utilities import logging_dispatcher 96 | return logging_dispatcher(map={}) 97 | 98 | def test_log_info(self): 99 | import logging 100 | inst = self._makeOne() 101 | logger = DummyLogger() 102 | inst.logger = logger 103 | inst.log_info('message', 'warning') 104 | self.assertEqual(logger.severity, logging.WARN) 105 | self.assertEqual(logger.message, 'message') 106 | 107 | class TestBadRequest(unittest.TestCase): 108 | 109 | def _makeOne(self): 110 | from waitress.utilities import BadRequest 111 | return BadRequest(1) 112 | 113 | def test_it(self): 114 | inst = self._makeOne() 115 | self.assertEqual(inst.body, 1) 116 | 117 | class DummyLogger(object): 118 | 119 | def log(self, severity, message): 120 | self.severity = severity 121 | self.message = message 122 | -------------------------------------------------------------------------------- /waitress/trigger.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001-2005 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | import asyncore 16 | import os 17 | import socket 18 | import errno 19 | import threading 20 | 21 | # Wake up a call to select() running in the main thread. 22 | # 23 | # This is useful in a context where you are using Medusa's I/O 24 | # subsystem to deliver data, but the data is generated by another 25 | # thread. Normally, if Medusa is in the middle of a call to 26 | # select(), new output data generated by another thread will have 27 | # to sit until the call to select() either times out or returns. 28 | # If the trigger is 'pulled' by another thread, it should immediately 29 | # generate a READ event on the trigger object, which will force the 30 | # select() invocation to return. 31 | # 32 | # A common use for this facility: letting Medusa manage I/O for a 33 | # large number of connections; but routing each request through a 34 | # thread chosen from a fixed-size thread pool. When a thread is 35 | # acquired, a transaction is performed, but output data is 36 | # accumulated into buffers that will be emptied more efficiently 37 | # by Medusa. [picture a server that can process database queries 38 | # rapidly, but doesn't want to tie up threads waiting to send data 39 | # to low-bandwidth connections] 40 | # 41 | # The other major feature provided by this class is the ability to 42 | # move work back into the main thread: if you call pull_trigger() 43 | # with a thunk argument, when select() wakes up and receives the 44 | # event it will call your thunk from within that thread. The main 45 | # purpose of this is to remove the need to wrap thread locks around 46 | # Medusa's data structures, which normally do not need them. [To see 47 | # why this is true, imagine this scenario: A thread tries to push some 48 | # new data onto a channel's outgoing data queue at the same time that 49 | # the main thread is trying to remove some] 50 | 51 | class _triggerbase(object): 52 | """OS-independent base class for OS-dependent trigger class.""" 53 | 54 | kind = None # subclass must set to "pipe" or "loopback"; used by repr 55 | 56 | def __init__(self): 57 | self._closed = False 58 | 59 | # `lock` protects the `thunks` list from being traversed and 60 | # appended to simultaneously. 61 | self.lock = threading.Lock() 62 | 63 | # List of no-argument callbacks to invoke when the trigger is 64 | # pulled. These run in the thread running the asyncore mainloop, 65 | # regardless of which thread pulls the trigger. 66 | self.thunks = [] 67 | 68 | def readable(self): 69 | return True 70 | 71 | def writable(self): 72 | return False 73 | 74 | def handle_connect(self): 75 | pass 76 | 77 | def handle_close(self): 78 | self.close() 79 | 80 | # Override the asyncore close() method, because it doesn't know about 81 | # (so can't close) all the gimmicks we have open. Subclass must 82 | # supply a _close() method to do platform-specific closing work. _close() 83 | # will be called iff we're not already closed. 84 | def close(self): 85 | if not self._closed: 86 | self._closed = True 87 | self.del_channel() 88 | self._close() # subclass does OS-specific stuff 89 | 90 | def pull_trigger(self, thunk=None): 91 | if thunk: 92 | with self.lock: 93 | self.thunks.append(thunk) 94 | self._physical_pull() 95 | 96 | def handle_read(self): 97 | try: 98 | self.recv(8192) 99 | except (OSError, socket.error): 100 | return 101 | with self.lock: 102 | for thunk in self.thunks: 103 | try: 104 | thunk() 105 | except: 106 | nil, t, v, tbinfo = asyncore.compact_traceback() 107 | self.log_info( 108 | 'exception in trigger thunk: (%s:%s %s)' % 109 | (t, v, tbinfo)) 110 | self.thunks = [] 111 | 112 | if os.name == 'posix': 113 | 114 | class trigger(_triggerbase, asyncore.file_dispatcher): 115 | kind = "pipe" 116 | 117 | def __init__(self, map): 118 | _triggerbase.__init__(self) 119 | r, self.trigger = self._fds = os.pipe() 120 | asyncore.file_dispatcher.__init__(self, r, map=map) 121 | 122 | def _close(self): 123 | for fd in self._fds: 124 | os.close(fd) 125 | self._fds = [] 126 | 127 | def _physical_pull(self): 128 | os.write(self.trigger, b'x') 129 | 130 | else: # pragma: no cover 131 | # Windows version; uses just sockets, because a pipe isn't select'able 132 | # on Windows. 133 | 134 | class trigger(_triggerbase, asyncore.dispatcher): 135 | kind = "loopback" 136 | 137 | def __init__(self, map): 138 | _triggerbase.__init__(self) 139 | 140 | # Get a pair of connected sockets. The trigger is the 'w' 141 | # end of the pair, which is connected to 'r'. 'r' is put 142 | # in the asyncore socket map. "pulling the trigger" then 143 | # means writing something on w, which will wake up r. 144 | 145 | w = socket.socket() 146 | # Disable buffering -- pulling the trigger sends 1 byte, 147 | # and we want that sent immediately, to wake up asyncore's 148 | # select() ASAP. 149 | w.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 150 | 151 | count = 0 152 | while True: 153 | count += 1 154 | # Bind to a local port; for efficiency, let the OS pick 155 | # a free port for us. 156 | # Unfortunately, stress tests showed that we may not 157 | # be able to connect to that port ("Address already in 158 | # use") despite that the OS picked it. This appears 159 | # to be a race bug in the Windows socket implementation. 160 | # So we loop until a connect() succeeds (almost always 161 | # on the first try). See the long thread at 162 | # http://mail.zope.org/pipermail/zope/2005-July/160433.html 163 | # for hideous details. 164 | a = socket.socket() 165 | a.bind(("127.0.0.1", 0)) 166 | connect_address = a.getsockname() # assigned (host, port) pair 167 | a.listen(1) 168 | try: 169 | w.connect(connect_address) 170 | break # success 171 | except socket.error as detail: 172 | if detail[0] != errno.WSAEADDRINUSE: 173 | # "Address already in use" is the only error 174 | # I've seen on two WinXP Pro SP2 boxes, under 175 | # Pythons 2.3.5 and 2.4.1. 176 | raise 177 | # (10048, 'Address already in use') 178 | # assert count <= 2 # never triggered in Tim's tests 179 | if count >= 10: # I've never seen it go above 2 180 | a.close() 181 | w.close() 182 | raise RuntimeError("Cannot bind trigger!") 183 | # Close `a` and try again. Note: I originally put a short 184 | # sleep() here, but it didn't appear to help or hurt. 185 | a.close() 186 | 187 | r, addr = a.accept() # r becomes asyncore's (self.)socket 188 | a.close() 189 | self.trigger = w 190 | asyncore.dispatcher.__init__(self, r, map=map) 191 | 192 | def _close(self): 193 | # self.socket is r, and self.trigger is w, from __init__ 194 | self.socket.close() 195 | self.trigger.close() 196 | 197 | def _physical_pull(self): 198 | self.trigger.send(b'x') 199 | -------------------------------------------------------------------------------- /waitress/utilities.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Utility functions 15 | """ 16 | 17 | import asyncore 18 | import errno 19 | import logging 20 | import os 21 | import re 22 | import stat 23 | import time 24 | import calendar 25 | 26 | logger = logging.getLogger('waitress') 27 | queue_logger = logging.getLogger('waitress.queue') 28 | 29 | def find_double_newline(s): 30 | """Returns the position just after a double newline in the given string.""" 31 | pos1 = s.find(b'\n\r\n') # One kind of double newline 32 | if pos1 >= 0: 33 | pos1 += 3 34 | pos2 = s.find(b'\n\n') # Another kind of double newline 35 | if pos2 >= 0: 36 | pos2 += 2 37 | 38 | if pos1 >= 0: 39 | if pos2 >= 0: 40 | return min(pos1, pos2) 41 | else: 42 | return pos1 43 | else: 44 | return pos2 45 | 46 | def concat(*args): 47 | return ''.join(args) 48 | 49 | def join(seq, field=' '): 50 | return field.join(seq) 51 | 52 | def group(s): 53 | return '(' + s + ')' 54 | 55 | short_days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat'] 56 | long_days = ['sunday', 'monday', 'tuesday', 'wednesday', 57 | 'thursday', 'friday', 'saturday'] 58 | 59 | short_day_reg = group(join(short_days, '|')) 60 | long_day_reg = group(join(long_days, '|')) 61 | 62 | daymap = {} 63 | for i in range(7): 64 | daymap[short_days[i]] = i 65 | daymap[long_days[i]] = i 66 | 67 | hms_reg = join(3 * [group('[0-9][0-9]')], ':') 68 | 69 | months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 70 | 'aug', 'sep', 'oct', 'nov', 'dec'] 71 | 72 | monmap = {} 73 | for i in range(12): 74 | monmap[months[i]] = i + 1 75 | 76 | months_reg = group(join(months, '|')) 77 | 78 | # From draft-ietf-http-v11-spec-07.txt/3.3.1 79 | # Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123 80 | # Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036 81 | # Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format 82 | 83 | # rfc822 format 84 | rfc822_date = join( 85 | [concat(short_day_reg, ','), # day 86 | group('[0-9][0-9]?'), # date 87 | months_reg, # month 88 | group('[0-9]+'), # year 89 | hms_reg, # hour minute second 90 | 'gmt' 91 | ], 92 | ' ' 93 | ) 94 | 95 | rfc822_reg = re.compile(rfc822_date) 96 | 97 | def unpack_rfc822(m): 98 | g = m.group 99 | return ( 100 | int(g(4)), # year 101 | monmap[g(3)], # month 102 | int(g(2)), # day 103 | int(g(5)), # hour 104 | int(g(6)), # minute 105 | int(g(7)), # second 106 | 0, 107 | 0, 108 | 0, 109 | ) 110 | 111 | # rfc850 format 112 | rfc850_date = join( 113 | [concat(long_day_reg, ','), 114 | join( 115 | [group('[0-9][0-9]?'), 116 | months_reg, 117 | group('[0-9]+') 118 | ], 119 | '-' 120 | ), 121 | hms_reg, 122 | 'gmt' 123 | ], 124 | ' ' 125 | ) 126 | 127 | rfc850_reg = re.compile(rfc850_date) 128 | # they actually unpack the same way 129 | def unpack_rfc850(m): 130 | g = m.group 131 | yr = g(4) 132 | if len(yr) == 2: 133 | yr = '19' + yr 134 | return ( 135 | int(yr), # year 136 | monmap[g(3)], # month 137 | int(g(2)), # day 138 | int(g(5)), # hour 139 | int(g(6)), # minute 140 | int(g(7)), # second 141 | 0, 142 | 0, 143 | 0 144 | ) 145 | 146 | # parsdate.parsedate - ~700/sec. 147 | # parse_http_date - ~1333/sec. 148 | 149 | weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] 150 | monthname = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 151 | 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] 152 | 153 | def build_http_date(when): 154 | year, month, day, hh, mm, ss, wd, y, z = time.gmtime(when) 155 | return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( 156 | weekdayname[wd], 157 | day, monthname[month], year, 158 | hh, mm, ss) 159 | 160 | def parse_http_date(d): 161 | d = d.lower() 162 | m = rfc850_reg.match(d) 163 | if m and m.end() == len(d): 164 | retval = int(calendar.timegm(unpack_rfc850(m))) 165 | else: 166 | m = rfc822_reg.match(d) 167 | if m and m.end() == len(d): 168 | retval = int(calendar.timegm(unpack_rfc822(m))) 169 | else: 170 | return 0 171 | return retval 172 | 173 | class logging_dispatcher(asyncore.dispatcher): 174 | logger = logger 175 | 176 | def log_info(self, message, type='info'): 177 | severity = { 178 | 'info': logging.INFO, 179 | 'warning': logging.WARN, 180 | 'error': logging.ERROR, 181 | } 182 | self.logger.log(severity.get(type, logging.INFO), message) 183 | 184 | def cleanup_unix_socket(path): 185 | try: 186 | st = os.stat(path) 187 | except OSError as exc: 188 | if exc.errno != errno.ENOENT: 189 | raise # pragma: no cover 190 | else: 191 | if stat.S_ISSOCK(st.st_mode): 192 | try: 193 | os.remove(path) 194 | except OSError: # pragma: no cover 195 | # avoid race condition error during tests 196 | pass 197 | 198 | class Error(object): 199 | 200 | def __init__(self, body): 201 | self.body = body 202 | 203 | class BadRequest(Error): 204 | code = 400 205 | reason = 'Bad Request' 206 | 207 | class RequestHeaderFieldsTooLarge(BadRequest): 208 | code = 431 209 | reason = 'Request Header Fields Too Large' 210 | 211 | class RequestEntityTooLarge(BadRequest): 212 | code = 413 213 | reason = 'Request Entity Too Large' 214 | 215 | class InternalServerError(Error): 216 | code = 500 217 | reason = 'Internal Server Error' 218 | --------------------------------------------------------------------------------