├── README ├── flake.py └── static ├── favicon.ico └── robots.txt /README: -------------------------------------------------------------------------------- 1 | Flake is an HTTP implementation of Twitter's Snowflake written in the Tornado async library. Snowflake is "a network service for generating unique ID numbers at high scale with some simple guarantees." The generated IDs are time plus worker id plus a sequence. Flake will send a 500 response if the system clock goes backwards or if the per millisecond sequence overflows. 2 | 3 | See also, http://github.com/twitter/snowflake. 4 | 5 | 6 | Usage: 7 | ./flake.py --worker_id=WORKER_ID --port=PORT 8 | 9 | Where WORKER_ID is a globally unique integer between 0 and 1023. 10 | 11 | The preferred network setup is to have multiple Flake servers and to connect randomly to one of them. Flake responses should be very quick (<10ms) so its reasonable to setup fallover after a short timeout. 12 | 13 | 14 | Requirements: 15 | 16 | Tornado, http://www.tornadoweb.org/ 17 | 18 | 19 | Example supervisord conf for a two core machine: 20 | 21 | [program:flake] 22 | command=/var/www/flake/flake.py --logging=warning --worker_id=%(process_num)d --port=80%(process_num)02d 23 | process_name=%(program_name)s%(process_num)d 24 | numprocs=2 25 | numprocs_start=0 ; be sure to set this offest such that worker_id is unique across all machines in your Flake cluster 26 | autostart=true 27 | 28 | 29 | Speed: 30 | Flake is reasonably fast. Two Flake processes behind nginx on a 2.4 GHz Core i5 Macbook Pro generate 3500 IDs/sec with a 99th %ile of 11ms (25 concurrent requests). 31 | -------------------------------------------------------------------------------- /flake.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2010 Formspring 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | import json 18 | import sys 19 | import tornado.httpserver 20 | import tornado.ioloop 21 | import tornado.options 22 | import tornado.web 23 | 24 | from time import time 25 | from tornado.options import define, options 26 | 27 | define("port", default=8888, help="run on the given port", type=int) 28 | define("worker_id", help="globally unique worker_id between 0 and 1023", type=int) 29 | 30 | 31 | class IDHandler(tornado.web.RequestHandler): 32 | max_time = int(time() * 1000) 33 | sequence = 0 34 | worker_id = False 35 | epoch = 1259193600000 # 2009-11-26 36 | 37 | def get(self): 38 | curr_time = int(time() * 1000) 39 | 40 | if curr_time < IDHandler.max_time: 41 | # stop handling requests til we've caught back up 42 | StatsHandler.errors += 1 43 | raise tornado.web.HTTPError(500, 'Clock went backwards! %d < %d' % (curr_time, IDHandler.max_time)) 44 | 45 | if curr_time > IDHandler.max_time: 46 | IDHandler.sequence = 0 47 | IDHandler.max_time = curr_time 48 | 49 | IDHandler.sequence += 1 50 | if IDHandler.sequence > 4095: 51 | # Sequence overflow, bail out 52 | StatsHandler.errors += 1 53 | raise tornado.web.HTTPError(500, 'Sequence Overflow: %d' % IDHandler.sequence) 54 | 55 | generated_id = ((curr_time - IDHandler.epoch) << 22) + (IDHandler.worker_id << 12) + IDHandler.sequence 56 | 57 | self.set_header("Content-Type", "text/plain") 58 | self.write(str(generated_id)) 59 | self.flush() # avoid ETag, etc generation 60 | 61 | StatsHandler.generated_ids += 1 62 | 63 | 64 | class StatsHandler(tornado.web.RequestHandler): 65 | generated_ids = 0 66 | errors = 0 67 | flush_time = time() 68 | 69 | def get(self): 70 | stats = { 71 | 'timestamp': time(), 72 | 'generated_ids': StatsHandler.generated_ids, 73 | 'errors': StatsHandler.errors, 74 | 'max_time_ms': IDHandler.max_time, 75 | 'worker_id': IDHandler.worker_id, 76 | 'time_since_flush': time() - StatsHandler.flush_time, 77 | } 78 | 79 | # Get values and reset 80 | if self.get_argument('flush', False): 81 | StatsHandler.generated_ids = 0 82 | StatsHandler.errors = 0 83 | StatsHandler.flush_time = time() 84 | 85 | self.set_header("Content-Type", "text/plain") 86 | self.write(json.dumps(stats)) 87 | 88 | 89 | def main(): 90 | tornado.options.parse_command_line() 91 | 92 | if 'worker_id' not in options: 93 | print 'missing --worker_id argument, see %s --help' % sys.argv[0] 94 | sys.exit() 95 | 96 | if not 0 <= options.worker_id < 1024: 97 | print 'invalid worker id, must be between 0 and 1023' 98 | sys.exit() 99 | 100 | IDHandler.worker_id = options.worker_id 101 | 102 | application = tornado.web.Application([ 103 | (r"/", IDHandler), 104 | (r"/stats", StatsHandler), 105 | ], static_path="./static") 106 | http_server = tornado.httpserver.HTTPServer(application) 107 | http_server.listen(options.port) 108 | tornado.ioloop.IOLoop.instance().start() 109 | 110 | 111 | if __name__ == "__main__": 112 | main() 113 | -------------------------------------------------------------------------------- /static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formspring/flake/04d46342e6a1770d19e01776ff014ab6effed41e/static/favicon.ico -------------------------------------------------------------------------------- /static/robots.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formspring/flake/04d46342e6a1770d19e01776ff014ab6effed41e/static/robots.txt --------------------------------------------------------------------------------