├── log └── placeholder ├── stratum ├── version.py ├── __init__.py ├── storage.py ├── event_handler.py ├── connection_registry.py ├── stats.py ├── logger.py ├── websocket_transport.py ├── custom_exceptions.py ├── semaphore.py ├── settings.py ├── helpers.py ├── example_service.py ├── jsonical.py ├── socksclient.py ├── irc.py ├── signature.py ├── server.py ├── socket_transport.py ├── config_default.py ├── pubsub.py ├── http_transport.py ├── services.py └── protocol.py ├── setup.py ├── launcher.tac ├── LICENSE ├── contrib ├── websocket.html └── haproxy.conf ├── client2.py ├── tests └── test_socket_transport.py ├── README ├── client.py └── distribute_setup.py /log/placeholder: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /stratum/version.py: -------------------------------------------------------------------------------- 1 | VERSION='0.2.15' 2 | -------------------------------------------------------------------------------- /stratum/__init__.py: -------------------------------------------------------------------------------- 1 | from server import setup 2 | -------------------------------------------------------------------------------- /stratum/storage.py: -------------------------------------------------------------------------------- 1 | #class StorageFactory(object): 2 | 3 | class Storage(object): 4 | #def __new__(self, session_id): 5 | # pass 6 | 7 | def __init__(self): 8 | self.__services = {} 9 | self.session = None 10 | 11 | def get(self, service_type, vendor, default_object): 12 | self.__services.setdefault(service_type, {}) 13 | self.__services[service_type].setdefault(vendor, default_object) 14 | return self.__services[service_type][vendor] 15 | 16 | def __repr__(self): 17 | return str(self.__services) -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from distribute_setup import use_setuptools 3 | use_setuptools() 4 | 5 | #python setup.py sdist upload 6 | 7 | from setuptools import setup 8 | from stratum import version 9 | 10 | setup(name='stratum', 11 | version=version.VERSION, 12 | description='Stratum server implementation based on Twisted', 13 | author='slush', 14 | author_email='info@bitcion.cz', 15 | url='http://blog.bitcoin.cz/stratum', 16 | packages=['stratum',], 17 | py_modules=['distribute_setup',], 18 | zip_safe=False, 19 | install_requires=['twisted', 'ecdsa', 'autobahn',] 20 | ) 21 | -------------------------------------------------------------------------------- /launcher.tac: -------------------------------------------------------------------------------- 1 | from twisted.python import log 2 | import stratum 3 | import stratum.settings as settings 4 | 5 | # This variable is used as an application handler by twistd 6 | application = stratum.setup() 7 | 8 | from twisted.internet import reactor 9 | 10 | def heartbeat(): 11 | log.msg('heartbeat') 12 | reactor.callLater(60, heartbeat) 13 | 14 | if settings.DEBUG: 15 | reactor.callLater(0, heartbeat) 16 | 17 | # Load all services from service_repository module. 18 | try: 19 | import service_repository 20 | except ImportError: 21 | print "***** Is service_repository missing? Add service_repository module to your python path!" -------------------------------------------------------------------------------- /stratum/event_handler.py: -------------------------------------------------------------------------------- 1 | import custom_exceptions 2 | from twisted.internet import defer 3 | from services import wrap_result_object 4 | 5 | class GenericEventHandler(object): 6 | def _handle_event(self, msg_method, msg_params, connection_ref): 7 | return defer.maybeDeferred(wrap_result_object, self.handle_event(msg_method, msg_params, connection_ref)) 8 | 9 | def handle_event(self, msg_method, msg_params, connection_ref): 10 | '''In most cases you'll only need to overload this method.''' 11 | print "Other side called method", msg_method, "with params", msg_params 12 | raise custom_exceptions.MethodNotFoundException("Method '%s' not implemented" % msg_method) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Stratum framework - Python implementation of Stratum protocol 2 | Copyright (C) 2012 Marek Palatinus 3 | 4 | Stratum is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | Stratum is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with Foobar. If not, see . 16 | -------------------------------------------------------------------------------- /stratum/connection_registry.py: -------------------------------------------------------------------------------- 1 | import weakref 2 | from twisted.internet import reactor 3 | from services import GenericService 4 | 5 | class ConnectionRegistry(object): 6 | __connections = weakref.WeakKeyDictionary() 7 | 8 | @classmethod 9 | def add_connection(cls, conn): 10 | cls.__connections[conn] = True 11 | 12 | @classmethod 13 | def remove_connection(cls, conn): 14 | try: 15 | del cls.__connections[conn] 16 | except: 17 | print "Warning: Cannot remove connection from ConnectionRegistry" 18 | 19 | @classmethod 20 | def get_session(cls, conn): 21 | if isinstance(conn, weakref.ref): 22 | conn = conn() 23 | 24 | if isinstance(conn, GenericService): 25 | conn = conn.connection_ref() 26 | 27 | if conn == None: 28 | return None 29 | 30 | return conn.get_session() 31 | 32 | @classmethod 33 | def iterate(cls): 34 | return cls.__connections.iterkeyrefs() 35 | 36 | def dump_connections(): 37 | for x in ConnectionRegistry.iterate(): 38 | c = x() 39 | c.transport.write('cus') 40 | print '!!!', c 41 | reactor.callLater(5, dump_connections) 42 | 43 | #reactor.callLater(0, dump_connections) 44 | -------------------------------------------------------------------------------- /stratum/stats.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logger 3 | log = logger.get_logger('stats') 4 | 5 | class PeerStats(object): 6 | '''Stub for server statistics''' 7 | counter = 0 8 | changes = 0 9 | 10 | @classmethod 11 | def client_connected(cls, ip): 12 | cls.counter += 1 13 | cls.changes += 1 14 | 15 | cls.print_stats() 16 | 17 | @classmethod 18 | def client_disconnected(cls, ip): 19 | cls.counter -= 1 20 | cls.changes += 1 21 | 22 | cls.print_stats() 23 | 24 | @classmethod 25 | def print_stats(cls): 26 | if cls.counter and float(cls.changes) / cls.counter < 0.05: 27 | # Print connection stats only when more than 28 | # 5% connections change to avoid log spam 29 | return 30 | 31 | log.info("%d peers connected, state changed %d times" % (cls.counter, cls.changes)) 32 | cls.changes = 0 33 | 34 | @classmethod 35 | def get_connected_clients(cls): 36 | return cls.counter 37 | 38 | ''' 39 | class CpuStats(object): 40 | start_time = time.time() 41 | 42 | @classmethod 43 | def get_time(cls): 44 | diff = time.time() - cls.start_time 45 | return resource.getrusage(resource.RUSAGE_SELF)[0] / diff 46 | ''' -------------------------------------------------------------------------------- /contrib/websocket.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 38 | 39 | 40 |
41 | Run WebSocket 42 |
43 | 44 | 45 | -------------------------------------------------------------------------------- /stratum/logger.py: -------------------------------------------------------------------------------- 1 | '''Simple wrapper around python's logging package''' 2 | 3 | import os 4 | import logging 5 | from twisted.python import log as twisted_log 6 | 7 | import settings 8 | 9 | ''' 10 | class Logger(object): 11 | def debug(self, msg): 12 | twisted_log.msg(msg) 13 | 14 | def info(self, msg): 15 | twisted_log.msg(msg) 16 | 17 | def warning(self, msg): 18 | twisted_log.msg(msg) 19 | 20 | def error(self, msg): 21 | twisted_log.msg(msg) 22 | 23 | def critical(self, msg): 24 | twisted_log.msg(msg) 25 | ''' 26 | 27 | def get_logger(name): 28 | logger = logging.getLogger(name) 29 | logger.addHandler(stream_handler) 30 | logger.setLevel(getattr(logging, settings.LOGLEVEL)) 31 | 32 | if settings.LOGFILE != None: 33 | logger.addHandler(file_handler) 34 | 35 | logger.debug("Logging initialized") 36 | return logger 37 | #return Logger() 38 | 39 | if settings.DEBUG: 40 | fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(module)s.%(funcName)s # %(message)s") 41 | else: 42 | fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s # %(message)s") 43 | 44 | if settings.LOGFILE != None: 45 | file_handler = logging.FileHandler(os.path.join(settings.LOGDIR, settings.LOGFILE)) 46 | file_handler.setFormatter(fmt) 47 | 48 | stream_handler = logging.StreamHandler() 49 | stream_handler.setFormatter(fmt) -------------------------------------------------------------------------------- /stratum/websocket_transport.py: -------------------------------------------------------------------------------- 1 | from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory 2 | from protocol import Protocol 3 | from event_handler import GenericEventHandler 4 | 5 | class WebsocketServerProtocol(WebSocketServerProtocol, Protocol): 6 | def connectionMade(self): 7 | WebSocketServerProtocol.connectionMade(self) 8 | Protocol.connectionMade(self) 9 | 10 | def connectionLost(self, reason): 11 | WebSocketServerProtocol.connectionLost(self, reason) 12 | Protocol.connectionLost(self, reason) 13 | 14 | def onMessage(self, msg, is_binary): 15 | Protocol.dataReceived(self, msg) 16 | 17 | def transport_write(self, data): 18 | self.sendMessage(data, False) 19 | 20 | class WebsocketTransportFactory(WebSocketServerFactory): 21 | def __init__(self, port, is_secure=False, debug=False, signing_key=None, signing_id=None, 22 | event_handler=GenericEventHandler): 23 | self.debug = debug 24 | self.signing_key = signing_key 25 | self.signing_id = signing_id 26 | self.protocol = WebsocketServerProtocol 27 | self.event_handler = event_handler 28 | 29 | if is_secure: 30 | uri = "wss://0.0.0.0:%d" % port 31 | else: 32 | uri = "ws://0.0.0.0:%d" % port 33 | 34 | WebSocketServerFactory.__init__(self, uri) 35 | 36 | # P.S. There's not Websocket client implementation yet 37 | # P.P.S. And it probably won't be for long time...' 38 | -------------------------------------------------------------------------------- /stratum/custom_exceptions.py: -------------------------------------------------------------------------------- 1 | class ProtocolException(Exception): 2 | pass 3 | 4 | class TransportException(Exception): 5 | pass 6 | 7 | class ServiceException(Exception): 8 | code = -2 9 | 10 | class UnauthorizedException(ServiceException): 11 | pass 12 | 13 | class SignatureException(ServiceException): 14 | code = -21 15 | 16 | class PubsubException(ServiceException): 17 | pass 18 | 19 | class AlreadySubscribedException(PubsubException): 20 | pass 21 | 22 | class IrcClientException(Exception): 23 | pass 24 | 25 | class SigningNotAvailableException(SignatureException): 26 | code = -21 27 | 28 | class UnknownSignatureIdException(SignatureException): 29 | code = -22 30 | 31 | class UnknownSignatureAlgorithmException(SignatureException): 32 | code = -22 33 | 34 | class SignatureVerificationFailedException(SignatureException): 35 | code = -23 36 | 37 | class MissingServiceTypeException(ServiceException): 38 | code = -2 39 | 40 | class MissingServiceVendorException(ServiceException): 41 | code = -2 42 | 43 | class MissingServiceIsDefaultException(ServiceException): 44 | code = -2 45 | 46 | class DefaultServiceAlreadyExistException(ServiceException): 47 | code = -2 48 | 49 | class ServiceNotFoundException(ServiceException): 50 | code = -2 51 | 52 | class MethodNotFoundException(ServiceException): 53 | code = -3 54 | 55 | class FeeRequiredException(ServiceException): 56 | code = -10 57 | 58 | class TimeoutServiceException(ServiceException): 59 | pass 60 | 61 | class RemoteServiceException(Exception): 62 | pass -------------------------------------------------------------------------------- /client2.py: -------------------------------------------------------------------------------- 1 | '''The simplest non-twisted client using HTTP polling''' 2 | import urllib2 3 | import time 4 | 5 | n = 1 6 | data = '{"id": 1, "method": "example.pubsub.subscribe", "params": [1]}'+"\n" 7 | data += '{"id": 2, "method": "example.ping", "params": ["cus"]}'+"\n" 8 | #data += '{"id": 3, "method": "example.synchronous", "params": [5]}'+"\n" 9 | #data += '{"id": 4, "method": "example.synchronous", "params": [5]}'+"\n" 10 | #data += '{"id": 5, "method": "example.synchronous", "params": [5]}'+"\n" 11 | 12 | ''' 13 | data = '{"id": 1, "method": "blockchain.block.ping", "params": ["test"]}'+"\n" 14 | data += '{"id": 2, "method": "firstbits.resolve", "params": ["1marek"]}'+"\n" 15 | data += '{"id": 3, "method": "blockchain.block.ping", "params": ["test2"]}'+"\n" 16 | data += '{"id": 4, "method": "txradar.lookup", "params": ["202e0d0ef7b1299a2193a7aa7bc58161789d72b62948b6005f2a4f190302740c"]}'+"\n" 17 | ''' 18 | 19 | try: 20 | headers = {'cookie': open('cookie.txt', 'r').read().strip(),} 21 | except: 22 | headers = {} 23 | 24 | headers['content-type'] = 'application/stratum' 25 | #headers['x-callback-url'] = 'http://localhost:20000' 26 | 27 | s = time.time() 28 | 29 | for x in range(n): 30 | r = urllib2.Request('http://california.stratum.bitcoin.cz:8000', data, headers) 31 | # r = urllib2.Request('http://localhost:8000', data, headers) 32 | resp = urllib2.urlopen(r) 33 | 34 | for h in resp.headers: 35 | print h, resp.headers[h] 36 | if h == 'set-cookie': 37 | open('cookie.txt', 'w').write(resp.headers[h]) 38 | 39 | print resp.read() 40 | print float(n) / (time.time() - s) 41 | -------------------------------------------------------------------------------- /stratum/semaphore.py: -------------------------------------------------------------------------------- 1 | from twisted.internet import defer 2 | 3 | class Semaphore: 4 | """A semaphore for event driven systems.""" 5 | 6 | def __init__(self, tokens): 7 | self.waiting = [] 8 | self.tokens = tokens 9 | self.limit = tokens 10 | 11 | def is_locked(self): 12 | return (bool)(not self.tokens) 13 | 14 | def acquire(self): 15 | """Attempt to acquire the token. 16 | 17 | @return Deferred which returns on token acquisition. 18 | """ 19 | assert self.tokens >= 0 20 | d = defer.Deferred() 21 | if not self.tokens: 22 | self.waiting.append(d) 23 | else: 24 | self.tokens = self.tokens - 1 25 | d.callback(self) 26 | return d 27 | 28 | def release(self): 29 | """Release the token. 30 | 31 | Should be called by whoever did the acquire() when the shared 32 | resource is free. 33 | """ 34 | assert self.tokens < self.limit 35 | self.tokens = self.tokens + 1 36 | if self.waiting: 37 | # someone is waiting to acquire token 38 | self.tokens = self.tokens - 1 39 | d = self.waiting.pop(0) 40 | d.callback(self) 41 | 42 | def _releaseAndReturn(self, r): 43 | self.release() 44 | return r 45 | 46 | def run(self, f, *args, **kwargs): 47 | """Acquire token, run function, release token. 48 | 49 | @return Deferred of function result. 50 | """ 51 | d = self.acquire() 52 | d.addCallback(lambda r: defer.maybeDeferred(f, *args, 53 | **kwargs).addBoth(self._releaseAndReturn)) 54 | return d 55 | -------------------------------------------------------------------------------- /stratum/settings.py: -------------------------------------------------------------------------------- 1 | def setup(): 2 | ''' 3 | This will import modules config_default and config and move their variables 4 | into current module (variables in config have higher priority than config_default). 5 | Thanks to this, you can import settings anywhere in the application and you'll get 6 | actual application settings. 7 | 8 | This config is related to server side. You don't need config.py if you 9 | want to use client part only. 10 | ''' 11 | 12 | def read_values(cfg): 13 | for varname in cfg.__dict__.keys(): 14 | if varname.startswith('__'): 15 | continue 16 | 17 | value = getattr(cfg, varname) 18 | yield (varname, value) 19 | 20 | import config_default 21 | 22 | try: 23 | import config 24 | except ImportError: 25 | # Custom config not presented, but we can still use defaults 26 | config = None 27 | 28 | import sys 29 | module = sys.modules[__name__] 30 | 31 | for name,value in read_values(config_default): 32 | module.__dict__[name] = value 33 | 34 | changes = {} 35 | if config: 36 | for name,value in read_values(config): 37 | if value != module.__dict__.get(name, None): 38 | changes[name] = value 39 | module.__dict__[name] = value 40 | 41 | if module.__dict__['DEBUG'] and changes: 42 | print "----------------" 43 | print "Custom settings:" 44 | for k, v in changes.items(): 45 | if 'passw' in k.lower(): 46 | print k, ": ********" 47 | else: 48 | print k, ":", v 49 | print "----------------" 50 | 51 | setup() 52 | -------------------------------------------------------------------------------- /tests/test_socket_transport.py: -------------------------------------------------------------------------------- 1 | from twisted.trial import unittest 2 | from twisted.internet import reactor 3 | from twisted.internet import defer 4 | import time 5 | 6 | import sys 7 | sys.path.append('../') 8 | 9 | from socket_transport import SocketTransportClientFactory 10 | 11 | HOSTNAME='localhost' 12 | PORT=3333 13 | 14 | class TcpTransportTestCase(unittest.TestCase): 15 | 16 | def _connect(self, hostname, port, on_connect, on_disconnect): 17 | # Try to connect to remote server 18 | return SocketTransportClientFactory(hostname, port, 19 | allow_trusted=True, 20 | allow_untrusted=False, 21 | debug=False, 22 | signing_key=None, 23 | signing_id=None, 24 | on_connect=on_connect, 25 | on_disconnect=on_disconnect, 26 | is_reconnecting=False) 27 | 28 | @defer.inlineCallbacks 29 | def setUp(self): 30 | self.on_connect = defer.Deferred() 31 | self.on_disconnect = defer.Deferred() 32 | self.f = self._connect(HOSTNAME, PORT, self.on_connect, self.on_disconnect) 33 | yield self.on_connect 34 | 35 | @defer.inlineCallbacks 36 | def tearDown(self): 37 | self.f.client.transport.loseConnection() 38 | yield self.on_disconnect 39 | 40 | @defer.inlineCallbacks 41 | def test_connection_timeout(self): 42 | on_connect = defer.Deferred() 43 | d = self.failUnlessFailure(on_connect, Exception) 44 | self._connect(HOSTNAME, 50999, on_connect, None) 45 | 46 | print "Please wait, this will take few seconds to complete..." 47 | yield d 48 | 49 | @defer.inlineCallbacks 50 | def test_ping(self): 51 | result = (yield self.f.rpc('node.ping', ['Some data',])) 52 | self.assertEquals(result, 'Some data', 'hu') 53 | 54 | @defer.inlineCallbacks 55 | def test_banner(self): 56 | result = (yield self.f.rpc('node.get_banner', [])) 57 | self.assertSubstring('', result) 58 | 59 | #if __name__ == '__main__': 60 | # unittest.main() -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | **!! Obsolete, not maintanied - DO NOT USE !!** 2 | I keep this repository alive and open for study and historical purposes only. 3 | It is no longer actively maintained (although there may be other forks that are still maintained). 4 | 5 | Description: 6 | ============ 7 | This is implementation of Stratum protocol for server and client side 8 | using asynchronous networking written in Python Twisted. 9 | 10 | Homepage: http://stratum.bitcoin.cz 11 | 12 | Contact to main developer: 13 | ========================== 14 | Email info at bitcoin.cz 15 | Nickname slush at bitcointalk.org forum 16 | 17 | Installation 18 | ============ 19 | 20 | Requirements: 21 | python 2.6 or 2.7 22 | linux-based system (should work on Mac OS too, not tested) 23 | 24 | Following instructions will work on Ubuntu & Debian*: 25 | 26 | a) From GIT, for developers 27 | git clone git://github.com/slush0/stratum.git 28 | sudo apt-get install python-dev 29 | sudo python setup.py develop 30 | 31 | b) From package, permanent install for production use 32 | sudo apt-get install python-dev 33 | sudo apt-get install python-setuptools 34 | sudo easy_install stratum 35 | 36 | *) Debian don't have a 'sudo' command, please do the installation 37 | process as a root user. 38 | 39 | Configuration 40 | ============= 41 | 42 | a) Basic configuration 43 | Copy config_default.py to config.py 44 | Edit at least those values: HOSTNAME, BITCOIN_TRUSTED_* 45 | 46 | b) Message signatures 47 | For enabling message signatures, generate server's ECDSA key by 48 | python signature.py > signing_key.pem 49 | and fill correct values to SIGNING_KEY and SIGNING_ID (config.py) 50 | 51 | c) Creating keys for SSL-based transports 52 | For all SSL-based transports (HTTPS, WSS, ...) you'll need private key 53 | and certificate file. You can use certificates from any authority or you can 54 | generate self-signed certificates, which is helpful at least for testing. 55 | 56 | Following script will generate self-signed SSL certificate: 57 | 58 | #!/bin/bash 59 | openssl genrsa -des3 -out server.key 1024 60 | openssl req -new -key server.key -out server.csr 61 | cp server.key server.key.org 62 | openssl rsa -in server.key.org -out server.key 63 | openssl x509 -req -in server.csr -signkey server.key -out server.crt 64 | 65 | Then you have to fill SSL_PRIVKEY and SSL_CACERT in config file with 66 | values 'server.key' and 'server.crt'. 67 | 68 | Startup 69 | ======= 70 | Start devel server: 71 | twistd -ny launcher.tac 72 | 73 | Devel server *without* lowlevel messages of Twisted: 74 | twistd -ny launcher.tac -l log/twistd.log 75 | 76 | Running in production 77 | ===================== 78 | TODO: Guide for running twistd as a daemon, init scripts 79 | TODO: Loadbalancing and port redirecting using haproxy 80 | TODO: Tunelling on 80/443 using stunnel 81 | Any volunteer for this ^ ? 82 | -------------------------------------------------------------------------------- /contrib/haproxy.conf: -------------------------------------------------------------------------------- 1 | # Under construction 2 | # TODO FIXME 3 | 4 | global 5 | maxconn 32768 # concurrent connections limit 6 | user nobody 7 | spread-checks 5 8 | 9 | defaults 10 | mode http 11 | option abortonclose 12 | no option accept-invalid-http-request 13 | no option accept-invalid-http-response 14 | option allbackups 15 | option forwardfor except 127.0.0.1 header X-Forwarded-For 16 | option redispatch 17 | retries 3 18 | option tcp-smart-connect 19 | 20 | 21 | frontend all 0.0.0.0:80 22 | # proxy HTTP 23 | default_backend nginx 24 | 25 | # proxy true WebSockets and socket.io fallbacks 26 | acl is_websocket hdr(upgrade) -i websocket 27 | acl is_websocket hdr_beg(host) -i ws 28 | acl is_websocket hdr_end(host) -i chat.obrool.com 29 | 30 | # N.B. the best is to place socket.io.min.js under production static server 31 | # so that it won't interfere here 32 | acl is_websocket path_beg /socket.io/ 33 | use_backend websockets if is_websocket 34 | 35 | # TODO: how to vary depending on backend? 36 | # 37 | # client must send the data within 25 seconds. should equal server timeout 38 | # TODO: long upload? route to another server? 39 | timeout client 1d #25s 40 | #nginx 41 | backend nginx 42 | balance roundrobin 43 | option forwardfor # This sets X-Forwarded-For 44 | server nginx 127.0.0.1:81 weight 1 maxconn 1024 check 45 | 46 | # timeouts 47 | option forceclose 48 | # server must be contacted within 5 seconds 49 | timeout connect 5s 50 | # all headers must arrive within 3 seconds 51 | timeout http-request 3s 52 | # server must respond within 25 seconds. should equal client timeout 53 | timeout server 25s 54 | # 55 | # don't close connection for 1 ms 56 | # helps to glue several short requests in one session 57 | # N.B. >10 can dramatically slow down short requests 58 | # 59 | timeout http-keep-alive 1 60 | 61 | backend websockets 62 | # workers 63 | default-server weight 50 maxqueue 16384 maxconn 16384 slowstart 1000 inter 5000 fastinter 500 downinter 10000 rise 2 fall 3 64 | # 65 | # CONFIG: enlist workers. 66 | # you should copy the list from above section 67 | # 68 | server s1 127.0.0.1:8003 69 | 70 | # balance the load 71 | balance roundrobin 72 | 73 | # cookie persistence 74 | cookie SERVERID insert indirect nocache 75 | # timeouts 76 | #option forceclose 77 | timeout queue 5s 78 | # support long-polling: socket.io uses circa 20-25 second requests, lets double the time 79 | timeout connect 60s 80 | timeout server 60s 81 | -------------------------------------------------------------------------------- /stratum/helpers.py: -------------------------------------------------------------------------------- 1 | from zope.interface import implements 2 | from twisted.internet import defer 3 | from twisted.internet import reactor 4 | from twisted.internet.protocol import Protocol 5 | from twisted.web.iweb import IBodyProducer 6 | from twisted.web.client import Agent 7 | from twisted.web.http_headers import Headers 8 | 9 | import settings 10 | 11 | class ResponseCruncher(Protocol): 12 | '''Helper for get_page()''' 13 | def __init__(self, finished): 14 | self.finished = finished 15 | self.response = "" 16 | 17 | def dataReceived(self, data): 18 | self.response += data 19 | 20 | def connectionLost(self, reason): 21 | self.finished.callback(self.response) 22 | 23 | class StringProducer(object): 24 | '''Helper for get_page()''' 25 | implements(IBodyProducer) 26 | 27 | def __init__(self, body): 28 | self.body = body 29 | self.length = len(body) 30 | 31 | def startProducing(self, consumer): 32 | consumer.write(self.body) 33 | return defer.succeed(None) 34 | 35 | def pauseProducing(self): 36 | pass 37 | 38 | def stopProducing(self): 39 | pass 40 | 41 | @defer.inlineCallbacks 42 | def get_page(url, method='GET', payload=None, headers=None): 43 | '''Downloads the page from given URL, using asynchronous networking''' 44 | agent = Agent(reactor) 45 | 46 | producer = None 47 | if payload: 48 | producer = StringProducer(payload) 49 | 50 | _headers = {'User-Agent': [settings.USER_AGENT,]} 51 | if headers: 52 | for key, value in headers.items(): 53 | _headers[key] = [value,] 54 | 55 | response = (yield agent.request( 56 | method, 57 | str(url), 58 | Headers(_headers), 59 | producer)) 60 | 61 | #for h in response.headers.getAllRawHeaders(): 62 | # print h 63 | 64 | try: 65 | finished = defer.Deferred() 66 | (yield response).deliverBody(ResponseCruncher(finished)) 67 | except: 68 | raise Exception("Downloading page '%s' failed" % url) 69 | 70 | defer.returnValue((yield finished)) 71 | 72 | @defer.inlineCallbacks 73 | def ask_old_server(method, *args): 74 | '''Perform request in old protocol to electrum servers. 75 | This is deprecated, used only for proxying some calls.''' 76 | import urllib 77 | import ast 78 | 79 | # Hack for methods without arguments 80 | if not len(args): 81 | args = ['',] 82 | 83 | res = (yield get_page('http://electrum.bitcoin.cz/electrum.php', method='POST', 84 | headers={"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}, 85 | payload=urllib.urlencode({'q': repr([method,] + list(args))}))) 86 | 87 | try: 88 | data = ast.literal_eval(res) 89 | except SyntaxError: 90 | print "Data received from server:", res 91 | raise Exception("Corrupted data from old electrum server") 92 | defer.returnValue(data) 93 | -------------------------------------------------------------------------------- /stratum/example_service.py: -------------------------------------------------------------------------------- 1 | from twisted.internet import defer 2 | from twisted.internet import reactor 3 | from twisted.names import client 4 | import random 5 | import time 6 | 7 | from services import GenericService, signature, synchronous 8 | import pubsub 9 | 10 | import logger 11 | log = logger.get_logger('example') 12 | 13 | class ExampleService(GenericService): 14 | service_type = 'example' 15 | service_vendor = 'Stratum' 16 | is_default = True 17 | 18 | def hello_world(self): 19 | return "Hello world!" 20 | hello_world.help_text = "Returns string 'Hello world!'" 21 | hello_world.params = [] 22 | 23 | @signature 24 | def ping(self, payload): 25 | return payload 26 | ping.help_text = "Returns signed message with the payload given by the client." 27 | ping.params = [('payload', 'mixed', 'This payload will be sent back to the client.'),] 28 | 29 | @synchronous 30 | def synchronous(self, how_long): 31 | '''This can use blocking calls, because it runs in separate thread''' 32 | for _ in range(int(how_long)): 33 | time.sleep(1) 34 | return 'Request finished in %d seconds' % how_long 35 | synchronous.help_text = "Run time consuming algorithm in server's threadpool and return the result when it finish." 36 | synchronous.params = [('how_long', 'int', 'For how many seconds the algorithm should run.'),] 37 | 38 | def throw_exception(self): 39 | raise Exception("Some error") 40 | throw_exception.help_text = "Throw an exception and send error result to the client." 41 | throw_exception.params = [] 42 | 43 | @signature 44 | def throw_signed_exception(self): 45 | raise Exception("Some error") 46 | throw_signed_exception.help_text = "Throw an exception and send signed error result to the client." 47 | throw_signed_exception.params = [] 48 | 49 | class TimeSubscription(pubsub.Subscription): 50 | event = 'example.pubsub.time_event' 51 | 52 | def process(self, t): 53 | # Process must return list of parameters for notification 54 | # or None if notification should not be send 55 | if t % self.params.get('period', 1) == 0: 56 | return (t,) 57 | 58 | def after_subscribe(self, _): 59 | # Some objects want to fire up notification or other 60 | # action directly after client subscribes. 61 | # after_subscribe is the right place for such logic 62 | pass 63 | 64 | class PubsubExampleService(GenericService): 65 | service_type = 'example.pubsub' 66 | service_vendor = 'Stratum' 67 | is_default = True 68 | 69 | def _setup(self): 70 | self._emit_time_event() 71 | 72 | @pubsub.subscribe 73 | def subscribe(self, period): 74 | return TimeSubscription(period=period) 75 | subscribe.help_text = "Subscribe client for receiving current server's unix timestamp." 76 | subscribe.params = [('period', 'int', 'Broadcast to the client only if timestamp%period==0. Use 1 for receiving an event in every second.'),] 77 | 78 | @pubsub.unsubscribe 79 | def unsubscribe(self, subscription_key):#period): 80 | return subscription_key 81 | unsubscribe.help_text = "Stop broadcasting unix timestampt to the client." 82 | unsubscribe.params = [('subscription_key', 'string', 'Key obtained by calling of subscribe method.'),] 83 | 84 | def _emit_time_event(self): 85 | # This will emit a publish event, 86 | # so all subscribed clients will receive 87 | # the notification 88 | 89 | t = time.time() 90 | TimeSubscription.emit(int(t)) 91 | reactor.callLater(1, self._emit_time_event) 92 | 93 | # Let's print some nice stats 94 | cnt = pubsub.Pubsub.get_subscription_count('example.pubsub.time_event') 95 | if cnt: 96 | log.info("Example event emitted in %.03f sec to %d subscribers" % (time.time() - t, cnt)) -------------------------------------------------------------------------------- /client.py: -------------------------------------------------------------------------------- 1 | from twisted.internet import reactor 2 | from twisted.internet import defer 3 | 4 | #import custom_exceptions 5 | from stratum.socket_transport import SocketTransportClientFactory 6 | from stratum.services import GenericService, ServiceEventHandler 7 | 8 | import time 9 | 10 | class PubsubReceivingService(GenericService): 11 | service_type = 'example.pubsub' 12 | service_vendor = 'custom' 13 | is_default = True 14 | 15 | def time_event(self, t): 16 | print "Received notification for the time_event !!!", t 17 | 18 | @defer.inlineCallbacks 19 | def unsubscribe(f, event, key): 20 | print "Unsubscribed:", (yield f.rpc('example.pubsub.unsubscribe', [key,])) 21 | reactor.stop() 22 | 23 | @defer.inlineCallbacks 24 | def main(): 25 | debug = False 26 | 27 | # Try to connect to remote server 28 | d = defer.Deferred() 29 | hostname = 'california.stratum.bitcoin.cz' 30 | #hostname = 'localhost' 31 | f = SocketTransportClientFactory(hostname, 3333, 32 | allow_trusted=True, 33 | allow_untrusted=False, 34 | debug=debug, 35 | signing_key=None, 36 | signing_id=None, 37 | on_connect=d, 38 | event_handler=ServiceEventHandler) 39 | yield d # Wait to on_connect event 40 | 41 | (event, subscription_key) = (yield f.subscribe('example.pubsub.subscribe', [1,])) 42 | print "Subscribed:", event, subscription_key 43 | reactor.callLater(3, unsubscribe, f, event, subscription_key) 44 | 45 | print (yield f.rpc('discovery.list_services', [])) 46 | #print (yield f.rpc('discovery.list_methods', ['example'])) 47 | #print (yield f.rpc('discovery.list_params', ['example.ping'])) 48 | 49 | print (yield f.rpc('example.ping', ['testing payload'])) 50 | ''' 51 | s = time.time() 52 | x = 10 53 | for x in range(x): 54 | (yield f.rpc('node.ping', ['hi',])) 55 | 56 | print float(x) / (time.time() - s) 57 | ''' 58 | #print (yield f.rpc('blockchain.transaction.broadcast', ['01000000016e71be76a49eac1d1f55113e4a581ea21a33e94a17453f6a0f3624db0b43b101000000008b48304502207e1fd0a8a8051fb21561df5d9d0e3ad0f3ced2b51e90ff1cebc85406a654cfaf022100bc7d3cf9a93959ff80953272d45fbd5cbdb8800874db0858783547ecef19c2ce014104e6a069738d8e8491a8abd3bed7d303c9b2dc3792173a18483653036fd74a5100fc6ee327b6a82b3df79005f101b88496988fa414af32df11fff3e96d53d26d03ffffffff0180969800000000001976a914e1c9b052561cf0a1da9ee3175df7d5a2d7ff7dd488ac00000000'])) 59 | 60 | ''' 61 | print (yield f.rpc_multi([ 62 | ['blockchain.block[Electrum].get_blocknum', [], False], 63 | ['node.get_peers', [], False], 64 | ['firstbits.create', ['1MarekMKDKRb6PEeHeVuiCGayk9avyBGBB',], False], 65 | ['firstbits.create', ['1MarekMKDKRb6PEeHeVuiCGayk9avyBGBB',], False], 66 | ['node.ping', ['test'], False], 67 | ])) 68 | ''' 69 | 70 | ''' 71 | print (yield f.rpc('blockchain.block.get_blocknum', [])) 72 | #print (yield f.rpc('node.get_peers', [])) 73 | 74 | try: 75 | # Example of full RPC call, including proper exception handling 76 | print (yield f.rpc('firstbits[firstbits.com].ping', ['nazdar',])) 77 | except custom_exceptions.RemoteServiceException as exc: 78 | print "RPC call failed", str(exc) 79 | 80 | # Example of service discover, this will print all known services, their vendors 81 | # and available methods on remote server 82 | print (yield f.rpc('discovery.list_services', [])) 83 | print (yield f.rpc('discovery.list_vendors', ['firstbits',])) 84 | print (yield f.rpc('discovery.list_methods', ['firstbits[firstbits.com]',])) 85 | 86 | # Example call of firstbits service 87 | print (yield f.rpc('firstbits.resolve', ['1MarekM',])) 88 | print (yield f.rpc('firstbits.create', ['1MarekMKDKRb6PEeHeVuiCGayk9avyBGBB',])) 89 | ''' 90 | #reactor.stop() 91 | 92 | main() 93 | reactor.run() 94 | -------------------------------------------------------------------------------- /stratum/jsonical.py: -------------------------------------------------------------------------------- 1 | # Copyright 2009 New England Biolabs 2 | # 3 | # This file is part of the nebgbhist package released under the MIT license. 4 | # 5 | r"""Canonical JSON serialization. 6 | 7 | Basic approaches for implementing canonical JSON serialization. 8 | 9 | Encoding basic Python object hierarchies:: 10 | 11 | >>> import jsonical 12 | >>> jsonical.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) 13 | '["foo",{"bar":["baz",null,1.0,2]}]' 14 | >>> print jsonical.dumps("\"foo\bar") 15 | "\"foo\bar" 16 | >>> print jsonical.dumps(u'\u1234') 17 | "\u1234" 18 | >>> print jsonical.dumps('\\') 19 | "\\" 20 | >>> print jsonical.dumps({"c": 0, "b": 0, "a": 0}) 21 | {"a":0,"b":0,"c":0} 22 | >>> from StringIO import StringIO 23 | >>> io = StringIO() 24 | >>> json.dump(['streaming API'], io) 25 | >>> io.getvalue() 26 | '["streaming API"]' 27 | 28 | Decoding JSON:: 29 | 30 | >>> import jsonical 31 | >>> jsonical.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') 32 | [u'foo', {u'bar': [u'baz', None, Decimal('1.0'), 2]}] 33 | >>> jsonical.loads('"\\"foo\\bar"') 34 | u'"foo\x08ar' 35 | >>> from StringIO import StringIO 36 | >>> io = StringIO('["streaming API"]') 37 | >>> jsonical.load(io) 38 | [u'streaming API'] 39 | 40 | Using jsonical from the shell to canonicalize: 41 | 42 | $ echo '{"json":"obj","bar":2.333333}' | python -mjsonical 43 | {"bar":2.333333,"json":"obj"} 44 | $ echo '{1.2:3.4}' | python -mjson.tool 45 | Expecting property name: line 1 column 2 (char 2) 46 | 47 | """ 48 | import datetime 49 | import decimal 50 | import sys 51 | import types 52 | import unicodedata 53 | 54 | try: 55 | import json 56 | except ImportError: 57 | import simplejson as json 58 | 59 | class Encoder(json.JSONEncoder): 60 | def __init__(self, *args, **kwargs): 61 | kwargs.pop("sort_keys", None) 62 | super(Encoder, self).__init__(sort_keys=True, *args, **kwargs) 63 | 64 | def default(self, obj): 65 | """This is slightly different than json.JSONEncoder.default(obj) 66 | in that it should returned the serialized representation of the 67 | passed object, not a serializable representation. 68 | """ 69 | if isinstance(obj, (datetime.date, datetime.time, datetime.datetime)): 70 | return '"%s"' % obj.isoformat() 71 | elif isinstance(obj, unicode): 72 | return '"%s"' % unicodedata.normalize('NFD', obj).encode('utf-8') 73 | elif isinstance(obj, decimal.Decimal): 74 | return str(obj) 75 | return super(Encoder, self).default(obj) 76 | 77 | def _iterencode_default(self, o, markers=None): 78 | yield self.default(o) 79 | 80 | def dump(obj, fp, indent=None): 81 | return json.dump(obj, fp, separators=(',', ':'), indent=indent, cls=Encoder) 82 | 83 | def dumps(obj, indent=None): 84 | return json.dumps(obj, separators=(',', ':'), indent=indent, cls=Encoder) 85 | 86 | class Decoder(json.JSONDecoder): 87 | def raw_decode(self, s, **kw): 88 | obj, end = super(Decoder, self).raw_decode(s, **kw) 89 | if isinstance(obj, types.StringTypes): 90 | obj = unicodedata.normalize('NFD', unicode(obj)) 91 | return obj, end 92 | 93 | def load(fp): 94 | return json.load(fp, cls=Decoder, parse_float=decimal.Decimal) 95 | 96 | def loads(s): 97 | return json.loads(s, cls=Decoder, parse_float=decimal.Decimal) 98 | 99 | def tool(): 100 | infile = sys.stdin 101 | outfile = sys.stdout 102 | if len(sys.argv) > 1: 103 | infile = open(sys.argv[1], 'rb') 104 | if len(sys.argv) > 2: 105 | outfile = open(sys.argv[2], 'wb') 106 | if len(sys.argv) > 3: 107 | raise SystemExit("{0} [infile [outfile]]".format(sys.argv[0])) 108 | try: 109 | obj = load(infile) 110 | except ValueError, e: 111 | raise SystemExit(e) 112 | dump(obj, outfile) 113 | outfile.write('\n') 114 | 115 | if __name__ == '__main__': 116 | tool() -------------------------------------------------------------------------------- /stratum/socksclient.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2011-2012, Linus Nordberg 2 | # Taken from https://github.com/ln5/twisted-socks/ 3 | 4 | import socket 5 | import struct 6 | from zope.interface import implements 7 | from twisted.internet import defer 8 | from twisted.internet.interfaces import IStreamClientEndpoint 9 | from twisted.internet.protocol import Protocol, ClientFactory 10 | from twisted.internet.endpoints import _WrappingFactory 11 | 12 | class SOCKSError(Exception): 13 | def __init__(self, val): 14 | self.val = val 15 | def __str__(self): 16 | return repr(self.val) 17 | 18 | class SOCKSv4ClientProtocol(Protocol): 19 | buf = '' 20 | 21 | def SOCKSConnect(self, host, port): 22 | # only socksv4a for now 23 | ver = 4 24 | cmd = 1 # stream connection 25 | user = '\x00' 26 | dnsname = '' 27 | try: 28 | addr = socket.inet_aton(host) 29 | except socket.error: 30 | addr = '\x00\x00\x00\x01' 31 | dnsname = '%s\x00' % host 32 | msg = struct.pack('!BBH', ver, cmd, port) + addr + user + dnsname 33 | self.transport.write(msg) 34 | 35 | def verifySocksReply(self, data): 36 | """ 37 | Return True on success, False on need-more-data. 38 | Raise SOCKSError on request rejected or failed. 39 | """ 40 | if len(data) < 8: 41 | return False 42 | if ord(data[0]) != 0: 43 | self.transport.loseConnection() 44 | raise SOCKSError((1, "bad data")) 45 | status = ord(data[1]) 46 | if status != 0x5a: 47 | self.transport.loseConnection() 48 | raise SOCKSError((status, "request not granted: %d" % status)) 49 | return True 50 | 51 | def isSuccess(self, data): 52 | self.buf += data 53 | return self.verifySocksReply(self.buf) 54 | 55 | def connectionMade(self): 56 | self.SOCKSConnect(self.postHandshakeEndpoint._host, 57 | self.postHandshakeEndpoint._port) 58 | 59 | def dataReceived(self, data): 60 | if self.isSuccess(data): 61 | # Build protocol from provided factory and transfer control to it. 62 | self.transport.protocol = self.postHandshakeFactory.buildProtocol( 63 | self.transport.getHost()) 64 | self.transport.protocol.transport = self.transport 65 | self.transport.protocol.connected = 1 66 | self.transport.protocol.connectionMade() 67 | self.handshakeDone.callback(self.transport.getPeer()) 68 | 69 | class SOCKSv4ClientFactory(ClientFactory): 70 | protocol = SOCKSv4ClientProtocol 71 | 72 | def buildProtocol(self, addr): 73 | r=ClientFactory.buildProtocol(self, addr) 74 | r.postHandshakeEndpoint = self.postHandshakeEndpoint 75 | r.postHandshakeFactory = self.postHandshakeFactory 76 | r.handshakeDone = self.handshakeDone 77 | return r 78 | 79 | class SOCKSWrapper(object): 80 | implements(IStreamClientEndpoint) 81 | factory = SOCKSv4ClientFactory 82 | 83 | def __init__(self, reactor, host, port, endpoint): 84 | self._host = host 85 | self._port = port 86 | self._reactor = reactor 87 | self._endpoint = endpoint 88 | 89 | def connect(self, protocolFactory): 90 | """ 91 | Return a deferred firing when the SOCKS connection is established. 92 | """ 93 | 94 | try: 95 | # Connect with an intermediate SOCKS factory/protocol, 96 | # which then hands control to the provided protocolFactory 97 | # once a SOCKS connection has been established. 98 | f = self.factory() 99 | f.postHandshakeEndpoint = self._endpoint 100 | f.postHandshakeFactory = protocolFactory 101 | f.handshakeDone = defer.Deferred() 102 | wf = _WrappingFactory(f) 103 | self._reactor.connectTCP(self._host, self._port, wf) 104 | return f.handshakeDone 105 | except: 106 | return defer.fail() -------------------------------------------------------------------------------- /stratum/irc.py: -------------------------------------------------------------------------------- 1 | from twisted.words.protocols import irc 2 | from twisted.internet import reactor, protocol 3 | import random 4 | import string 5 | 6 | import custom_exceptions 7 | import logger 8 | log = logger.get_logger('irc') 9 | 10 | # Reference to open IRC connection 11 | _connection = None 12 | 13 | def get_connection(): 14 | if _connection: 15 | return _connection 16 | 17 | raise custom_exceptions.IrcClientException("IRC not connected") 18 | 19 | class IrcLurker(irc.IRCClient): 20 | def connectionMade(self): 21 | irc.IRCClient.connectionMade(self) 22 | self.peers = {} 23 | 24 | global _connection 25 | _connection = self 26 | 27 | def get_peers(self): 28 | return self.peers.values() 29 | 30 | def connectionLost(self, reason): 31 | irc.IRCClient.connectionLost(self, reason) 32 | 33 | global _connection 34 | _connection = None 35 | 36 | def signedOn(self): 37 | self.join(self.factory.channel) 38 | 39 | def joined(self, channel): 40 | log.info('Joined %s' % channel) 41 | 42 | #def dataReceived(self, data): 43 | # print data 44 | # irc.IRCClient.dataReceived(self, data.replace('\r', '')) 45 | 46 | def privmsg(self, user, channel, msg): 47 | user = user.split('!', 1)[0] 48 | 49 | if channel == self.nickname or msg.startswith(self.nickname + ":"): 50 | log.info("'%s': %s" % (user, msg)) 51 | return 52 | 53 | #def action(self, user, channel, msg): 54 | # user = user.split('!', 1)[0] 55 | # print user, channel, msg 56 | 57 | def register(self, nickname, *args, **kwargs): 58 | self.setNick(nickname) 59 | self.sendLine("USER %s 0 * :%s" % (self.nickname, self.factory.hostname)) 60 | 61 | def irc_RPL_NAMREPLY(self, prefix, params): 62 | for nick in params[3].split(' '): 63 | if not nick.startswith('S_'): 64 | continue 65 | 66 | if nick == self.nickname: 67 | continue 68 | 69 | self.sendLine("WHO %s" % nick) 70 | 71 | def irc_RPL_WHOREPLY(self, prefix, params): 72 | nickname = params[5] 73 | hostname = params[7].split(' ', 1)[1] 74 | log.debug("New peer '%s' (%s)" % (hostname, nickname)) 75 | self.peers[nickname] = hostname 76 | 77 | def userJoined(self, nickname, channel): 78 | self.sendLine("WHO %s" % nickname) 79 | 80 | def userLeft(self, nickname, channel): 81 | self.userQuit(nickname) 82 | 83 | def userKicked(self, nickname, *args, **kwargs): 84 | self.userQuit(nickname) 85 | 86 | def userQuit(self, nickname, *args, **kwargs): 87 | try: 88 | hostname = self.peers[nickname] 89 | del self.peers[nickname] 90 | log.info("Peer '%s' (%s) disconnected" % (hostname, nickname)) 91 | except: 92 | pass 93 | 94 | #def irc_unknown(self, prefix, command, params): 95 | # print "UNKNOWN", prefix, command, params 96 | 97 | class IrcLurkerFactory(protocol.ClientFactory): 98 | def __init__(self, channel, nickname, hostname): 99 | self.channel = channel 100 | self.nickname = nickname 101 | self.hostname = hostname 102 | 103 | def _random_string(self, N): 104 | return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(N)) 105 | 106 | def buildProtocol(self, addr): 107 | p = IrcLurker() 108 | p.factory = self 109 | p.nickname = "S_%s_%s" % (self.nickname, self._random_string(5)) 110 | log.info("Using nickname '%s'" % p.nickname) 111 | return p 112 | 113 | def clientConnectionLost(self, connector, reason): 114 | """If we get disconnected, reconnect to server.""" 115 | log.error("Connection lost") 116 | reactor.callLater(10, connector.connect) 117 | 118 | def clientConnectionFailed(self, connector, reason): 119 | log.error("Connection failed") 120 | reactor.callLater(10, connector.connect) 121 | 122 | if __name__ == '__main__': 123 | # Example of using IRC bot 124 | reactor.connectTCP("irc.freenode.net", 6667, IrcLurkerFactory('#stratum-nodes', 'test', 'example.com')) 125 | reactor.run() -------------------------------------------------------------------------------- /stratum/signature.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | try: 3 | import ecdsa 4 | from ecdsa import curves 5 | except ImportError: 6 | print "ecdsa package not installed. Signing of messages not available." 7 | ecdsa = None 8 | 9 | import base64 10 | import hashlib 11 | import time 12 | 13 | import jsonical 14 | import json 15 | import custom_exceptions 16 | 17 | if ecdsa: 18 | # secp256k1, http://www.oid-info.com/get/1.3.132.0.10 19 | _p = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2FL 20 | _r = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141L 21 | _b = 0x0000000000000000000000000000000000000000000000000000000000000007L 22 | _a = 0x0000000000000000000000000000000000000000000000000000000000000000L 23 | _Gx = 0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798L 24 | _Gy = 0x483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8L 25 | curve_secp256k1 = ecdsa.ellipticcurve.CurveFp(_p, _a, _b) 26 | generator_secp256k1 = ecdsa.ellipticcurve.Point(curve_secp256k1, _Gx, _Gy, _r) 27 | oid_secp256k1 = (1,3,132,0,10) 28 | SECP256k1 = ecdsa.curves.Curve("SECP256k1", curve_secp256k1, generator_secp256k1, oid_secp256k1 ) 29 | 30 | # Register SECP256k1 to ecdsa library 31 | curves.curves.append(SECP256k1) 32 | 33 | def generate_keypair(): 34 | if not ecdsa: 35 | raise custom_exceptions.SigningNotAvailableException("ecdsa not installed") 36 | 37 | private_key = ecdsa.SigningKey.generate(curve=SECP256k1) 38 | public_key = private_key.get_verifying_key() 39 | return (private_key, public_key) 40 | 41 | def load_privkey_pem(filename): 42 | return ecdsa.SigningKey.from_pem(open(filename, 'r').read().strip()) 43 | 44 | def sign(privkey, data): 45 | if not ecdsa: 46 | raise custom_exceptions.SigningNotAvailableException("ecdsa not installed") 47 | 48 | hash = hashlib.sha256(data).digest() 49 | signature = privkey.sign_digest(hash, sigencode=ecdsa.util.sigencode_der) 50 | return base64.b64encode(signature) 51 | 52 | def verify(pubkey, signature, data): 53 | if not ecdsa: 54 | raise custom_exceptions.SigningNotAvailableException("ecdsa not installed") 55 | 56 | hash = hashlib.sha256(data).digest() 57 | sign = base64.b64decode(signature) 58 | try: 59 | return pubkey.verify_digest(sign, hash, sigdecode=ecdsa.util.sigdecode_der) 60 | except: 61 | return False 62 | 63 | def jsonrpc_dumps_sign(privkey, privkey_id, is_request, message_id, method='', params=[], result=None, error=None): 64 | '''Create the signature for given json-rpc data and returns signed json-rpc text stream''' 65 | 66 | # Build data object to sign 67 | sign_time = int(time.time()) 68 | data = {'method': method, 'params': params, 'result': result, 'error': error, 'sign_time': sign_time} 69 | 70 | # Serialize data to sign and perform signing 71 | txt = jsonical.dumps(data) 72 | signature = sign(privkey, txt) 73 | 74 | # Reconstruct final data object and put signature 75 | if is_request: 76 | data = {'id': message_id, 'method': method, 'params': params, 77 | 'sign': signature, 'sign_algo': 'ecdsa;SECP256k1', 'sign_id': privkey_id, 'sign_time': sign_time} 78 | else: 79 | data = {'id': message_id, 'result': result, 'error': error, 80 | 'sign': signature, 'sign_algo': 'ecdsa;SECP256k1', 'sign_id': privkey_id, 'sign_time': sign_time} 81 | 82 | # Return original data extended with signature 83 | return jsonical.dumps(data) 84 | 85 | def jsonrpc_loads_verify(pubkeys, txt): 86 | ''' 87 | Pubkeys is mapping (dict) of sign_id -> ecdsa public key. 88 | This method deserialize provided json-encoded data, load signature ID, perform the lookup for public key 89 | and check stored signature of the message. If signature is OK, returns message data. 90 | ''' 91 | data = json.loads(txt) 92 | signature_algo = data['sign_algo'] 93 | signature_id = data['sign_id'] 94 | signature_time = data['sign_time'] 95 | 96 | if signature_algo != 'ecdsa;SECP256k1': 97 | raise custom_exceptions.UnknownSignatureAlgorithmException("%s is not supported" % signature_algo) 98 | 99 | try: 100 | pubkey = pubkeys[signature_id] 101 | except KeyError: 102 | raise custom_exceptions.UnknownSignatureIdException("Public key for '%s' not found" % signature_id) 103 | 104 | signature = data['sign'] 105 | message_id = data['id'] 106 | method = data.get('method', '') 107 | params = data.get('params', []) 108 | result = data.get('result', None) 109 | error = data.get('error', None) 110 | 111 | # Build data object to verify 112 | data = {'method': method, 'params': params, 'result': result, 'error': error, 'sign_time': signature_time} 113 | txt = jsonical.dumps(data) 114 | 115 | if not verify(pubkey, signature, txt): 116 | raise custom_exceptions.SignatureVerificationFailedException("Signature doesn't match to given data") 117 | 118 | if method: 119 | # It's a request 120 | return {'id': message_id, 'method': method, 'params': params} 121 | 122 | else: 123 | # It's aresponse 124 | return {'id': message_id, 'result': result, 'error': error} 125 | 126 | if __name__ == '__main__': 127 | (private, public) = generate_keypair() 128 | print private.to_pem() 129 | -------------------------------------------------------------------------------- /stratum/server.py: -------------------------------------------------------------------------------- 1 | def setup(setup_event=None): 2 | try: 3 | from twisted.internet import epollreactor 4 | epollreactor.install() 5 | except ImportError: 6 | print "Failed to install epoll reactor, default reactor will be used instead." 7 | 8 | try: 9 | import settings 10 | except ImportError: 11 | print "***** Is configs.py missing? Maybe you want to copy and customize config_default.py?" 12 | 13 | from twisted.application import service 14 | application = service.Application("stratum-server") 15 | 16 | # Setting up logging 17 | from twisted.python.log import ILogObserver, FileLogObserver 18 | from twisted.python.logfile import DailyLogFile 19 | 20 | #logfile = DailyLogFile(settings.LOGFILE, settings.LOGDIR) 21 | #application.setComponent(ILogObserver, FileLogObserver(logfile).emit) 22 | 23 | if settings.ENABLE_EXAMPLE_SERVICE: 24 | import stratum.example_service 25 | 26 | if setup_event == None: 27 | setup_finalize(None, application) 28 | else: 29 | setup_event.addCallback(setup_finalize, application) 30 | 31 | return application 32 | 33 | def setup_finalize(event, application): 34 | 35 | from twisted.application import service, internet 36 | from twisted.internet import reactor, ssl 37 | from twisted.web.server import Site 38 | from twisted.python import log 39 | #from twisted.enterprise import adbapi 40 | import OpenSSL.SSL 41 | 42 | from services import ServiceEventHandler 43 | 44 | import socket_transport 45 | import http_transport 46 | import websocket_transport 47 | import irc 48 | 49 | from stratum import settings 50 | 51 | try: 52 | import signature 53 | signing_key = signature.load_privkey_pem(settings.SIGNING_KEY) 54 | except: 55 | print "Loading of signing key '%s' failed, protocol messages cannot be signed." % settings.SIGNING_KEY 56 | signing_key = None 57 | 58 | # Attach HTTPS Poll Transport service to application 59 | try: 60 | sslContext = ssl.DefaultOpenSSLContextFactory(settings.SSL_PRIVKEY, settings.SSL_CACERT) 61 | except OpenSSL.SSL.Error: 62 | sslContext = None 63 | print "Cannot initiate SSL context, are SSL_PRIVKEY or SSL_CACERT missing?" 64 | print "This will skip all SSL-based transports." 65 | 66 | # Set up thread pool size for service threads 67 | reactor.suggestThreadPoolSize(settings.THREAD_POOL_SIZE) 68 | 69 | if settings.LISTEN_SOCKET_TRANSPORT: 70 | # Attach Socket Transport service to application 71 | socket = internet.TCPServer(settings.LISTEN_SOCKET_TRANSPORT, 72 | socket_transport.SocketTransportFactory(debug=settings.DEBUG, 73 | signing_key=signing_key, 74 | signing_id=settings.SIGNING_ID, 75 | event_handler=ServiceEventHandler, 76 | tcp_proxy_protocol_enable=settings.TCP_PROXY_PROTOCOL)) 77 | socket.setServiceParent(application) 78 | 79 | # Build the HTTP interface 80 | httpsite = Site(http_transport.Root(debug=settings.DEBUG, signing_key=signing_key, signing_id=settings.SIGNING_ID, 81 | event_handler=ServiceEventHandler)) 82 | httpsite.sessionFactory = http_transport.HttpSession 83 | 84 | if settings.LISTEN_HTTP_TRANSPORT: 85 | # Attach HTTP Poll Transport service to application 86 | http = internet.TCPServer(settings.LISTEN_HTTP_TRANSPORT, httpsite) 87 | http.setServiceParent(application) 88 | 89 | if settings.LISTEN_HTTPS_TRANSPORT and sslContext: 90 | https = internet.SSLServer(settings.LISTEN_HTTPS_TRANSPORT, httpsite, contextFactory = sslContext) 91 | https.setServiceParent(application) 92 | 93 | if settings.LISTEN_WS_TRANSPORT: 94 | from autobahn.websocket import listenWS 95 | log.msg("Starting WS transport on %d" % settings.LISTEN_WS_TRANSPORT) 96 | ws = websocket_transport.WebsocketTransportFactory(settings.LISTEN_WS_TRANSPORT, 97 | debug=settings.DEBUG, 98 | signing_key=signing_key, 99 | signing_id=settings.SIGNING_ID, 100 | event_handler=ServiceEventHandler) 101 | listenWS(ws) 102 | 103 | if settings.LISTEN_WSS_TRANSPORT and sslContext: 104 | from autobahn.websocket import listenWS 105 | log.msg("Starting WSS transport on %d" % settings.LISTEN_WSS_TRANSPORT) 106 | wss = websocket_transport.WebsocketTransportFactory(settings.LISTEN_WSS_TRANSPORT, is_secure=True, 107 | debug=settings.DEBUG, 108 | signing_key=signing_key, 109 | signing_id=settings.SIGNING_ID, 110 | event_handler=ServiceEventHandler) 111 | listenWS(wss, contextFactory=sslContext) 112 | 113 | if settings.IRC_NICK: 114 | reactor.connectTCP(settings.IRC_SERVER, settings.IRC_PORT, irc.IrcLurkerFactory(settings.IRC_ROOM, settings.IRC_NICK, settings.IRC_HOSTNAME)) 115 | 116 | return event 117 | 118 | if __name__ == '__main__': 119 | print "This is not executable script. Try 'twistd -ny launcher.tac instead!" 120 | -------------------------------------------------------------------------------- /stratum/socket_transport.py: -------------------------------------------------------------------------------- 1 | from twisted.internet.protocol import ServerFactory 2 | from twisted.internet.protocol import ReconnectingClientFactory 3 | from twisted.internet import reactor, defer, endpoints 4 | 5 | import socksclient 6 | import custom_exceptions 7 | from protocol import Protocol, ClientProtocol 8 | from event_handler import GenericEventHandler 9 | 10 | import logger 11 | log = logger.get_logger('socket_transport') 12 | 13 | def sockswrapper(proxy, dest): 14 | endpoint = endpoints.TCP4ClientEndpoint(reactor, dest[0], dest[1]) 15 | return socksclient.SOCKSWrapper(reactor, proxy[0], proxy[1], endpoint) 16 | 17 | class SocketTransportFactory(ServerFactory): 18 | def __init__(self, debug=False, signing_key=None, signing_id=None, event_handler=GenericEventHandler, 19 | tcp_proxy_protocol_enable=False): 20 | self.debug = debug 21 | self.signing_key = signing_key 22 | self.signing_id = signing_id 23 | self.event_handler = event_handler 24 | self.protocol = Protocol 25 | 26 | # Read settings.TCP_PROXY_PROTOCOL documentation 27 | self.tcp_proxy_protocol_enable = tcp_proxy_protocol_enable 28 | 29 | class SocketTransportClientFactory(ReconnectingClientFactory): 30 | def __init__(self, host, port, allow_trusted=True, allow_untrusted=False, 31 | debug=False, signing_key=None, signing_id=None, 32 | is_reconnecting=True, proxy=None, 33 | event_handler=GenericEventHandler): 34 | self.debug = debug 35 | self.is_reconnecting = is_reconnecting 36 | self.signing_key = signing_key 37 | self.signing_id = signing_id 38 | self.client = None # Reference to open connection 39 | self.on_disconnect = defer.Deferred() 40 | self.on_connect = defer.Deferred() 41 | self.peers_trusted = {} 42 | self.peers_untrusted = {} 43 | self.main_host = (host, port) 44 | self.new_host = None 45 | self.proxy = proxy 46 | 47 | self.event_handler = event_handler 48 | self.protocol = ClientProtocol 49 | self.after_connect = [] 50 | 51 | self.connect() 52 | 53 | def connect(self): 54 | if self.proxy: 55 | self.timeout_handler = reactor.callLater(60, self.connection_timeout) 56 | sw = sockswrapper(self.proxy, self.main_host) 57 | sw.connect(self) 58 | else: 59 | self.timeout_handler = reactor.callLater(30, self.connection_timeout) 60 | reactor.connectTCP(self.main_host[0], self.main_host[1], self) 61 | 62 | ''' 63 | This shouldn't be a part of transport layer 64 | def add_peers(self, peers): 65 | # FIXME: Use this list when current connection fails 66 | for peer in peers: 67 | hash = "%s%s%s" % (peer['hostname'], peer['ipv4'], peer['ipv6']) 68 | 69 | which = self.peers_trusted if peer['trusted'] else self.peers_untrusted 70 | which[hash] = peer 71 | 72 | #print self.peers_trusted 73 | #print self.peers_untrusted 74 | ''' 75 | 76 | def connection_timeout(self): 77 | self.timeout_handler = None 78 | 79 | if self.client: 80 | return 81 | 82 | e = custom_exceptions.TransportException("SocketTransportClientFactory connection timed out") 83 | if not self.on_connect.called: 84 | d = self.on_connect 85 | self.on_connect = defer.Deferred() 86 | d.errback(e) 87 | 88 | else: 89 | raise e 90 | 91 | def rpc(self, method, params, *args, **kwargs): 92 | if not self.client: 93 | raise custom_exceptions.TransportException("Not connected") 94 | 95 | return self.client.rpc(method, params, *args, **kwargs) 96 | 97 | def subscribe(self, method, params, *args, **kwargs): 98 | ''' 99 | This is like standard RPC call, except that parameters are stored 100 | into after_connect list, so the same command will perform again 101 | on restored connection. 102 | ''' 103 | if not self.client: 104 | raise custom_exceptions.TransportException("Not connected") 105 | 106 | self.after_connect.append((method, params)) 107 | return self.client.rpc(method, params, *args, **kwargs) 108 | 109 | def reconnect(self, host=None, port=None, wait=None): 110 | '''Close current connection and start new one. 111 | If host or port specified, it will be used for new connection.''' 112 | 113 | new = list(self.main_host) 114 | if host: 115 | new[0] = host 116 | if port: 117 | new[1] = port 118 | self.new_host = tuple(new) 119 | 120 | if self.client and self.client.connected: 121 | if wait != None: 122 | self.delay = wait 123 | self.client.transport.connector.disconnect() 124 | 125 | def retry(self, connector=None): 126 | if not self.is_reconnecting: 127 | return 128 | 129 | if connector is None: 130 | if self.connector is None: 131 | raise ValueError("no connector to retry") 132 | else: 133 | connector = self.connector 134 | 135 | if self.new_host: 136 | # Switch to new host if any 137 | connector.host = self.new_host[0] 138 | connector.port = self.new_host[1] 139 | self.main_host = self.new_host 140 | self.new_host = None 141 | 142 | return ReconnectingClientFactory.retry(self, connector) 143 | 144 | def buildProtocol(self, addr): 145 | self.resetDelay() 146 | #if not self.is_reconnecting: raise 147 | return ReconnectingClientFactory.buildProtocol(self, addr) 148 | 149 | def clientConnectionLost(self, connector, reason): 150 | if self.is_reconnecting: 151 | log.debug(reason) 152 | ReconnectingClientFactory.clientConnectionLost(self, connector, reason) 153 | 154 | def clientConnectionFailed(self, connector, reason): 155 | if self.is_reconnecting: 156 | log.debug(reason) 157 | ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) 158 | 159 | -------------------------------------------------------------------------------- /stratum/config_default.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This is example configuration for Stratum server. 3 | Please rename it to config.py and fill correct values. 4 | ''' 5 | 6 | # ******************** GENERAL SETTINGS *************** 7 | 8 | # Enable some verbose debug (logging requests and responses). 9 | DEBUG = True 10 | 11 | # Destination for application logs, files rotated once per day. 12 | LOGDIR = 'log/' 13 | 14 | # Main application log file. 15 | LOGFILE = None #'stratum.log' 16 | 17 | # Possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL 18 | LOGLEVEL = 'DEBUG' 19 | 20 | # How many threads use for synchronous methods (services). 21 | # 30 is enough for small installation, for real usage 22 | # it should be slightly more, say 100-300. 23 | THREAD_POOL_SIZE = 30 24 | 25 | # RPC call throws TimeoutServiceException once total time since request has been 26 | # placed (time to delivery to client + time for processing on the client) 27 | # crosses _TOTAL (in second). 28 | # _TOTAL reflects the fact that not all transports deliver RPC requests to the clients 29 | # instantly, so request can wait some time in the buffer on server side. 30 | # NOT IMPLEMENTED YET 31 | #RPC_TIMEOUT_TOTAL = 600 32 | 33 | # RPC call throws TimeoutServiceException once client is processing request longer 34 | # than _PROCESS (in second) 35 | # NOT IMPLEMENTED YET 36 | #RPC_TIMEOUT_PROCESS = 30 37 | 38 | # Do you want to expose "example" service in server? 39 | # Useful for learning the server,you probably want to disable 40 | # this on production 41 | ENABLE_EXAMPLE_SERVICE = True 42 | 43 | # ******************** TRANSPORTS ********************* 44 | 45 | # Hostname or external IP to expose 46 | HOSTNAME = 'stratum.example.com' 47 | 48 | # Port used for Socket transport. Use 'None' for disabling the transport. 49 | LISTEN_SOCKET_TRANSPORT = 3333 50 | 51 | # Port used for HTTP Poll transport. Use 'None' for disabling the transport 52 | LISTEN_HTTP_TRANSPORT = 8000 53 | 54 | # Port used for HTTPS Poll transport 55 | LISTEN_HTTPS_TRANSPORT = 8001 56 | 57 | # Port used for WebSocket transport, 'None' for disabling WS 58 | LISTEN_WS_TRANSPORT = 8002 59 | 60 | # Port used for secure WebSocket, 'None' for disabling WSS 61 | LISTEN_WSS_TRANSPORT = 8003 62 | 63 | # ******************** SSL SETTINGS ****************** 64 | 65 | # Private key and certification file for SSL protected transports 66 | # You can find howto for generating self-signed certificate in README file 67 | SSL_PRIVKEY = 'server.key' 68 | SSL_CACERT = 'server.crt' 69 | 70 | # ******************** TCP SETTINGS ****************** 71 | 72 | # Enables support for socket encapsulation, which is compatible 73 | # with haproxy 1.5+. By enabling this, first line of received 74 | # data will represent some metadata about proxied stream: 75 | # PROXY \n 76 | # 77 | # Full specification: http://haproxy.1wt.eu/download/1.5/doc/proxy-protocol.txt 78 | TCP_PROXY_PROTOCOL = False 79 | 80 | # ******************** HTTP SETTINGS ***************** 81 | 82 | # Keepalive for HTTP transport sessions (at this time for both poll and push) 83 | # High value leads to higher memory usage (all sessions are stored in memory ATM). 84 | # Low value leads to more frequent session reinitializing (like downloading address history). 85 | HTTP_SESSION_TIMEOUT = 3600 # in seconds 86 | 87 | # Maximum number of messages (notifications, responses) waiting to delivery to HTTP Poll clients. 88 | # Buffer length is PER CONNECTION. High value will consume a lot of RAM, 89 | # short history will cause that in some edge cases clients won't receive older events. 90 | HTTP_BUFFER_LIMIT = 10000 91 | 92 | # User agent used in HTTP requests (for both HTTP transports and for proxy calls from services) 93 | USER_AGENT = 'Stratum/0.1' 94 | 95 | # Provide human-friendly user interface on HTTP transports for browsing exposed services. 96 | BROWSER_ENABLE = True 97 | 98 | # ******************** BITCOIND SETTINGS ************ 99 | 100 | # Hostname and credentials for one trusted Bitcoin node ("Satoshi's client"). 101 | # Stratum uses both P2P port (which is 8333 everytime) and RPC port 102 | BITCOIN_TRUSTED_HOST = '127.0.0.1' 103 | BITCOIN_TRUSTED_PORT = 8332 # RPC port 104 | BITCOIN_TRUSTED_USER = 'stratum' 105 | BITCOIN_TRUSTED_PASSWORD = '***somepassword***' 106 | 107 | # ******************** OTHER CORE SETTINGS ********************* 108 | # Use "echo -n '' | sha256sum | cut -f1 -d' ' " 109 | # for calculating SHA256 of your preferred password 110 | ADMIN_PASSWORD_SHA256 = None # Admin functionality is disabled 111 | #ADMIN_PASSWORD_SHA256 = '9e6c0c1db1e0dfb3fa5159deb4ecd9715b3c8cd6b06bd4a3ad77e9a8c5694219' # SHA256 of the password 112 | 113 | # IP from which admin calls are allowed. 114 | # Set None to allow admin calls from all IPs 115 | ADMIN_RESTRICT_INTERFACE = '127.0.0.1' 116 | 117 | # Use "./signature.py > signing_key.pem" to generate unique signing key for your server 118 | SIGNING_KEY = None # Message signing is disabled 119 | #SIGNING_KEY = 'signing_key.pem' 120 | 121 | # Origin of signed messages. Provide some unique string, 122 | # ideally URL where users can find some information about your identity 123 | SIGNING_ID = None 124 | #SIGNING_ID = 'stratum.somedomain.com' # Use custom string 125 | #SIGNING_ID = HOSTNAME # Use hostname as the signing ID 126 | 127 | # *********************** IRC / PEER CONFIGURATION ************* 128 | 129 | IRC_NICK = None # Skip IRC registration 130 | #IRC_NICK = "stratum" # Use nickname of your choice 131 | 132 | # Which hostname / external IP expose in IRC room 133 | # This should be official HOSTNAME for normal operation. 134 | IRC_HOSTNAME = HOSTNAME 135 | 136 | # Don't change this unless you're creating private Stratum cloud. 137 | IRC_SERVER = 'irc.freenode.net' 138 | IRC_ROOM = '#stratum-nodes' 139 | IRC_PORT = 6667 140 | 141 | # Hardcoded list of Stratum nodes for clients to switch when this node is not available. 142 | PEERS = [ 143 | { 144 | 'hostname': 'stratum.bitcoin.cz', 145 | 'trusted': True, # This node is trustworthy 146 | 'weight': -1, # Higher number means higher priority for selection. 147 | # -1 will work mostly as a backup when other servers won't work. 148 | # (IRC peers have weight=0 automatically). 149 | }, 150 | ] 151 | 152 | 153 | ''' 154 | DATABASE_DRIVER = 'MySQLdb' 155 | DATABASE_HOST = 'palatinus.cz' 156 | DATABASE_DBNAME = 'marekp_bitcointe' 157 | DATABASE_USER = 'marekp_bitcointe' 158 | DATABASE_PASSWORD = '**empty**' 159 | ''' 160 | -------------------------------------------------------------------------------- /stratum/pubsub.py: -------------------------------------------------------------------------------- 1 | import weakref 2 | from connection_registry import ConnectionRegistry 3 | import custom_exceptions 4 | import hashlib 5 | 6 | def subscribe(func): 7 | '''Decorator detect Subscription object in result and subscribe connection''' 8 | def inner(self, *args, **kwargs): 9 | subs = func(self, *args, **kwargs) 10 | return Pubsub.subscribe(self.connection_ref(), subs) 11 | return inner 12 | 13 | def unsubscribe(func): 14 | '''Decorator detect Subscription object in result and unsubscribe connection''' 15 | def inner(self, *args, **kwargs): 16 | subs = func(self, *args, **kwargs) 17 | if isinstance(subs, Subscription): 18 | return Pubsub.unsubscribe(self.connection_ref(), subscription=subs) 19 | else: 20 | return Pubsub.unsubscribe(self.connection_ref(), key=subs) 21 | return inner 22 | 23 | class Subscription(object): 24 | def __init__(self, event=None, **params): 25 | if hasattr(self, 'event'): 26 | if event: 27 | raise Exception("Event name already defined in Subscription object") 28 | else: 29 | if not event: 30 | raise Exception("Please define event name in constructor") 31 | else: 32 | self.event = event 33 | 34 | self.params = params # Internal parameters for subscription object 35 | self.connection_ref = None 36 | 37 | def process(self, *args, **kwargs): 38 | return args 39 | 40 | def get_key(self): 41 | '''This is an identifier for current subscription. It is sent to the client, 42 | so result should not contain any sensitive information.''' 43 | return hashlib.md5(str((self.event, self.params))).hexdigest() 44 | 45 | def get_session(self): 46 | '''Connection session may be useful in filter or process functions''' 47 | return self.connection_ref().get_session() 48 | 49 | @classmethod 50 | def emit(cls, *args, **kwargs): 51 | '''Shortcut for emiting this event to all subscribers.''' 52 | if not hasattr(cls, 'event'): 53 | raise Exception("Subscription.emit() can be used only for subclasses with filled 'event' class variable.") 54 | return Pubsub.emit(cls.event, *args, **kwargs) 55 | 56 | def emit_single(self, *args, **kwargs): 57 | '''Perform emit of this event just for current subscription.''' 58 | conn = self.connection_ref() 59 | if conn == None: 60 | # Connection is closed 61 | return 62 | 63 | payload = self.process(*args, **kwargs) 64 | if payload != None: 65 | if isinstance(payload, (tuple, list)): 66 | conn.writeJsonRequest(self.event, payload, is_notification=True) 67 | self.after_emit(*args, **kwargs) 68 | else: 69 | raise Exception("Return object from process() method must be list or None") 70 | 71 | def after_emit(self, *args, **kwargs): 72 | pass 73 | 74 | # Once function is defined, it will be called every time 75 | #def after_subscribe(self, _): 76 | # pass 77 | 78 | def __eq__(self, other): 79 | return (isinstance(other, Subscription) and other.get_key() == self.get_key()) 80 | 81 | def __ne__(self, other): 82 | return not self.__eq__(other) 83 | 84 | class Pubsub(object): 85 | __subscriptions = {} 86 | 87 | @classmethod 88 | def subscribe(cls, connection, subscription): 89 | if connection == None: 90 | raise custom_exceptions.PubsubException("Subscriber not connected") 91 | 92 | key = subscription.get_key() 93 | session = ConnectionRegistry.get_session(connection) 94 | if session == None: 95 | raise custom_exceptions.PubsubException("No session found") 96 | 97 | subscription.connection_ref = weakref.ref(connection) 98 | session.setdefault('subscriptions', {}) 99 | 100 | if key in session['subscriptions']: 101 | raise custom_exceptions.AlreadySubscribedException("This connection is already subscribed for such event.") 102 | 103 | session['subscriptions'][key] = subscription 104 | 105 | cls.__subscriptions.setdefault(subscription.event, weakref.WeakKeyDictionary()) 106 | cls.__subscriptions[subscription.event][subscription] = None 107 | 108 | if hasattr(subscription, 'after_subscribe'): 109 | if connection.on_finish != None: 110 | # If subscription is processed during the request, wait to 111 | # finish and then process the callback 112 | connection.on_finish.addCallback(subscription.after_subscribe) 113 | else: 114 | # If subscription is NOT processed during the request (any real use case?), 115 | # process callback instantly (better now than never). 116 | subscription.after_subscribe(True) 117 | 118 | # List of 2-tuples is prepared for future multi-subscriptions 119 | return ((subscription.event, key),) 120 | 121 | @classmethod 122 | def unsubscribe(cls, connection, subscription=None, key=None): 123 | if connection == None: 124 | raise custom_exceptions.PubsubException("Subscriber not connected") 125 | 126 | session = ConnectionRegistry.get_session(connection) 127 | if session == None: 128 | raise custom_exceptions.PubsubException("No session found") 129 | 130 | if subscription: 131 | key = subscription.get_key() 132 | 133 | try: 134 | # Subscription don't need to be removed from cls.__subscriptions, 135 | # because it uses weak reference there. 136 | del session['subscriptions'][key] 137 | except KeyError: 138 | print "Warning: Cannot remove subscription from connection session" 139 | return False 140 | 141 | return True 142 | 143 | @classmethod 144 | def get_subscription_count(cls, event): 145 | return len(cls.__subscriptions.get(event, {})) 146 | 147 | @classmethod 148 | def get_subscription(cls, connection, event, key=None): 149 | '''Return subscription object for given connection and event''' 150 | session = ConnectionRegistry.get_session(connection) 151 | if session == None: 152 | raise custom_exceptions.PubsubException("No session found") 153 | 154 | if key == None: 155 | sub = [ sub for sub in session.get('subscriptions', {}).values() if sub.event == event ] 156 | try: 157 | return sub[0] 158 | except IndexError: 159 | raise custom_exceptions.PubsubException("Not subscribed for event %s" % event) 160 | 161 | else: 162 | raise Exception("Searching subscriptions by key is not implemented yet") 163 | 164 | @classmethod 165 | def iterate_subscribers(cls, event): 166 | for subscription in cls.__subscriptions.get(event, weakref.WeakKeyDictionary()).iterkeyrefs(): 167 | subscription = subscription() 168 | if subscription == None: 169 | # Subscriber is no more connected 170 | continue 171 | 172 | yield subscription 173 | 174 | @classmethod 175 | def emit(cls, event, *args, **kwargs): 176 | for subscription in cls.iterate_subscribers(event): 177 | subscription.emit_single(*args, **kwargs) -------------------------------------------------------------------------------- /stratum/http_transport.py: -------------------------------------------------------------------------------- 1 | from twisted.web.resource import Resource 2 | from twisted.web.server import Request, Session, NOT_DONE_YET 3 | from twisted.internet import defer 4 | from twisted.python.failure import Failure 5 | import hashlib 6 | import json 7 | import string 8 | 9 | import helpers 10 | import semaphore 11 | #from storage import Storage 12 | from protocol import Protocol, RequestCounter 13 | from event_handler import GenericEventHandler 14 | import settings 15 | 16 | import logger 17 | log = logger.get_logger('http_transport') 18 | 19 | class Transport(object): 20 | def __init__(self, session_id, lock): 21 | self.buffer = [] 22 | self.session_id = session_id 23 | self.lock = lock 24 | self.push_url = None # None or full URL for HTTP Push 25 | self.peer = None 26 | 27 | # For compatibility with generic transport, not used in HTTP transport 28 | self.disconnecting = False 29 | 30 | def getPeer(self): 31 | return self.peer 32 | 33 | def write(self, data): 34 | if len(self.buffer) >= settings.HTTP_BUFFER_LIMIT: 35 | # Drop first (oldest) item in buffer 36 | # if buffer crossed allowed limit. 37 | # This isn't totally exact, because one record in buffer 38 | # can teoretically contains more than one message (divided by \n), 39 | # but current server implementation don't store responses in this way, 40 | # so counting exact number of messages will lead to unnecessary overhead. 41 | self.buffer.pop(0) 42 | 43 | self.buffer.append(data) 44 | 45 | if not self.lock.is_locked() and self.push_url: 46 | # Push the buffer to callback URL 47 | # TODO: Buffer responses and perform callbgitacks in batches 48 | self.push_buffer() 49 | 50 | def push_buffer(self): 51 | '''Push the content of the buffer into callback URL''' 52 | if not self.push_url: 53 | return 54 | 55 | # FIXME: Don't expect any response 56 | helpers.get_page(self.push_url, method='POST', 57 | headers={"content-type": "application/stratum", 58 | "x-session-id": self.session_id}, 59 | payload=self.fetch_buffer()) 60 | 61 | def fetch_buffer(self): 62 | ret = ''.join(self.buffer) 63 | self.buffer = [] 64 | return ret 65 | 66 | def set_push_url(self, url): 67 | self.push_url = url 68 | 69 | def monkeypatch_method(cls): 70 | '''Perform monkey patch for given class.''' 71 | def decorator(func): 72 | setattr(cls, func.__name__, func) 73 | return func 74 | return decorator 75 | 76 | @monkeypatch_method(Request) 77 | def getSession(self, sessionInterface=None, cookie_prefix='TWISTEDSESSION'): 78 | '''Monkey patch for Request object, providing backward-compatible 79 | getSession method which can handle custom cookie as a session ID 80 | (which is necessary for following Stratum protocol specs). 81 | Unfortunately twisted developers rejected named-cookie feature, 82 | which is pressing me into this ugly solution... 83 | 84 | TODO: Especially this would deserve some unit test to be sure it doesn't break 85 | in future twisted versions. 86 | ''' 87 | # Session management 88 | if not self.session: 89 | cookiename = string.join([cookie_prefix] + self.sitepath, "_") 90 | sessionCookie = self.getCookie(cookiename) 91 | if sessionCookie: 92 | try: 93 | self.session = self.site.getSession(sessionCookie) 94 | except KeyError: 95 | pass 96 | # if it still hasn't been set, fix it up. 97 | if not self.session: 98 | self.session = self.site.makeSession() 99 | self.addCookie(cookiename, self.session.uid, path='/') 100 | self.session.touch() 101 | if sessionInterface: 102 | return self.session.getComponent(sessionInterface) 103 | return self.session 104 | 105 | class HttpSession(Session): 106 | sessionTimeout = settings.HTTP_SESSION_TIMEOUT 107 | 108 | def __init__(self, *args, **kwargs): 109 | Session.__init__(self, *args, **kwargs) 110 | #self.storage = Storage() 111 | 112 | # Reference to connection object (Protocol instance) 113 | self.protocol = None 114 | 115 | # Synchronizing object for avoiding race condition on session 116 | self.lock = semaphore.Semaphore(1) 117 | 118 | # Output buffering 119 | self.transport = Transport(self.uid, self.lock) 120 | 121 | # Setup cleanup method on session expiration 122 | self.notifyOnExpire(lambda: HttpSession.on_expire(self)) 123 | 124 | @classmethod 125 | def on_expire(cls, sess_obj): 126 | # FIXME: Close protocol connection 127 | print "EXPIRING SESSION", sess_obj 128 | 129 | if sess_obj.protocol: 130 | sess_obj.protocol.connectionLost(Failure(Exception("HTTP session closed"))) 131 | 132 | sess_obj.protocol = None 133 | 134 | class Root(Resource): 135 | isLeaf = True 136 | 137 | def __init__(self, debug=False, signing_key=None, signing_id=None, 138 | event_handler=GenericEventHandler): 139 | Resource.__init__(self) 140 | self.signing_key = signing_key 141 | self.signing_id = signing_id 142 | self.debug = debug # This class acts as a 'factory', debug is used by Protocol 143 | self.event_handler = event_handler 144 | 145 | def render_GET(self, request): 146 | if not settings.BROWSER_ENABLE: 147 | return "Welcome to %s server. Use HTTP POST to talk with the server." % settings.USER_AGENT 148 | 149 | # TODO: Web browser 150 | return "Web browser not implemented yet" 151 | 152 | def render_OPTIONS(self, request): 153 | session = request.getSession(cookie_prefix='STRATUM_SESSION') 154 | 155 | request.setHeader('server', settings.USER_AGENT) 156 | request.setHeader('x-session-timeout', session.sessionTimeout) 157 | request.setHeader('access-control-allow-origin', '*') # Allow access from any other domain 158 | request.setHeader('access-control-allow-methods', 'POST, OPTIONS') 159 | request.setHeader('access-control-allow-headers', 'Content-Type') 160 | return '' 161 | 162 | def render_POST(self, request): 163 | session = request.getSession(cookie_prefix='STRATUM_SESSION') 164 | 165 | l = session.lock.acquire() 166 | l.addCallback(self._perform_request, request, session) 167 | return NOT_DONE_YET 168 | 169 | def _perform_request(self, _, request, session): 170 | request.setHeader('content-type', 'application/stratum') 171 | request.setHeader('server', settings.USER_AGENT) 172 | request.setHeader('x-session-timeout', session.sessionTimeout) 173 | request.setHeader('access-control-allow-origin', '*') # Allow access from any other domain 174 | 175 | # Update client's IP address 176 | session.transport.peer = request.getHost() 177 | 178 | # Although it isn't intuitive at all, request.getHeader reads request headers, 179 | # but request.setHeader (few lines above) writes response headers... 180 | if 'application/stratum' not in request.getHeader('content-type'): 181 | session.transport.write("%s\n" % json.dumps({'id': None, 'result': None, 'error': (-1, "Content-type must be 'application/stratum'. See http://stratum.bitcoin.cz for more info.", "")})) 182 | self._finish(None, request, session.transport, session.lock) 183 | return 184 | 185 | if not session.protocol: 186 | # Build a "protocol connection" 187 | proto = Protocol() 188 | proto.transport = session.transport 189 | proto.factory = self 190 | proto.connectionMade() 191 | session.protocol = proto 192 | else: 193 | proto = session.protocol 194 | 195 | # Update callback URL if presented 196 | callback_url = request.getHeader('x-callback-url') 197 | if callback_url != None: 198 | if callback_url == '': 199 | # Blank value of callback URL switches HTTP Push back to HTTP Poll 200 | session.transport.push_url = None 201 | else: 202 | session.transport.push_url = callback_url 203 | 204 | data = request.content.read() 205 | if data: 206 | counter = RequestCounter() 207 | counter.on_finish.addCallback(self._finish, request, session.transport, session.lock) 208 | proto.dataReceived(data, request_counter=counter) 209 | else: 210 | # Ping message (empty request) of HTTP Polling 211 | self._finish(None, request, session.transport, session.lock) 212 | 213 | 214 | @classmethod 215 | def _finish(cls, _, request, transport, lock): 216 | # First parameter is callback result; not used here 217 | data = transport.fetch_buffer() 218 | request.setHeader('content-length', len(data)) 219 | request.setHeader('content-md5', hashlib.md5(data).hexdigest()) 220 | request.setHeader('x-content-sha256', hashlib.sha256(data).hexdigest()) 221 | request.write(data) 222 | request.finish() 223 | lock.release() 224 | -------------------------------------------------------------------------------- /stratum/services.py: -------------------------------------------------------------------------------- 1 | from twisted.internet import defer, threads 2 | from twisted.python import log 3 | import hashlib 4 | import weakref 5 | import re 6 | 7 | import custom_exceptions 8 | 9 | VENDOR_RE = re.compile(r'\[(.*)\]') 10 | 11 | class ServiceEventHandler(object): # reimplements event_handler.GenericEventHandler 12 | def _handle_event(self, msg_method, msg_params, connection_ref): 13 | return ServiceFactory.call(msg_method, msg_params, connection_ref=connection_ref) 14 | 15 | class ResultObject(object): 16 | def __init__(self, result=None, sign=False, sign_algo=None, sign_id=None): 17 | self.result = result 18 | self.sign = sign 19 | self.sign_algo = sign_algo 20 | self.sign_id = sign_id 21 | 22 | def wrap_result_object(obj): 23 | def _wrap(o): 24 | if isinstance(o, ResultObject): 25 | return o 26 | return ResultObject(result=o) 27 | 28 | if isinstance(obj, defer.Deferred): 29 | # We don't have result yet, just wait for it and wrap it later 30 | obj.addCallback(_wrap) 31 | return obj 32 | 33 | return _wrap(obj) 34 | 35 | class ServiceFactory(object): 36 | registry = {} # Mapping service_type -> vendor -> cls 37 | 38 | @classmethod 39 | def _split_method(cls, method): 40 | '''Parses "some.service[vendor].method" string 41 | and returns 3-tuple with (service_type, vendor, rpc_method)''' 42 | 43 | # Splits the service type and method name 44 | (service_type, method_name) = method.rsplit('.', 1) 45 | vendor = None 46 | 47 | if '[' in service_type: 48 | # Use regular expression only when brackets found 49 | try: 50 | vendor = VENDOR_RE.search(service_type).group(1) 51 | service_type = service_type.replace('[%s]' % vendor, '') 52 | except: 53 | raise 54 | #raise custom_exceptions.ServiceNotFoundException("Invalid syntax in service name '%s'" % type_name[0]) 55 | 56 | return (service_type, vendor, method_name) 57 | 58 | @classmethod 59 | def call(cls, method, params, connection_ref=None): 60 | try: 61 | (service_type, vendor, func_name) = cls._split_method(method) 62 | except ValueError: 63 | raise custom_exceptions.MethodNotFoundException("Method name parsing failed. You *must* use format ., e.g. 'example.ping'") 64 | 65 | try: 66 | if func_name.startswith('_'): 67 | raise 68 | 69 | _inst = cls.lookup(service_type, vendor=vendor)() 70 | _inst.connection_ref = weakref.ref(connection_ref) 71 | func = _inst.__getattribute__(func_name) 72 | if not callable(func): 73 | raise 74 | except: 75 | raise custom_exceptions.MethodNotFoundException("Method '%s' not found for service '%s'" % (func_name, service_type)) 76 | 77 | def _run(func, *params): 78 | return wrap_result_object(func(*params)) 79 | 80 | # Returns Defer which will lead to ResultObject sometimes 81 | return defer.maybeDeferred(_run, func, *params) 82 | 83 | @classmethod 84 | def lookup(cls, service_type, vendor=None): 85 | # Lookup for service type provided by specific vendor 86 | if vendor: 87 | try: 88 | return cls.registry[service_type][vendor] 89 | except KeyError: 90 | raise custom_exceptions.ServiceNotFoundException("Class for given service type and vendor isn't registered") 91 | 92 | # Lookup for any vendor, prefer default one 93 | try: 94 | vendors = cls.registry[service_type] 95 | except KeyError: 96 | raise custom_exceptions.ServiceNotFoundException("Class for given service type isn't registered") 97 | 98 | last_found = None 99 | for _, _cls in vendors.items(): 100 | last_found = _cls 101 | if last_found.is_default: 102 | return last_found 103 | 104 | if not last_found: 105 | raise custom_exceptions.ServiceNotFoundException("Class for given service type isn't registered") 106 | 107 | return last_found 108 | 109 | @classmethod 110 | def register_service(cls, _cls, meta): 111 | # Register service class to ServiceFactory 112 | service_type = meta.get('service_type') 113 | service_vendor = meta.get('service_vendor') 114 | is_default = meta.get('is_default') 115 | 116 | if str(_cls.__name__) in ('GenericService',): 117 | # str() is ugly hack, but it is avoiding circular references 118 | return 119 | 120 | if not service_type: 121 | raise custom_exceptions.MissingServiceTypeException("Service class '%s' is missing 'service_type' property." % _cls) 122 | 123 | if not service_vendor: 124 | raise custom_exceptions.MissingServiceVendorException("Service class '%s' is missing 'service_vendor' property." % _cls) 125 | 126 | if is_default == None: 127 | raise custom_exceptions.MissingServiceIsDefaultException("Service class '%s' is missing 'is_default' property." % _cls) 128 | 129 | if is_default: 130 | # Check if there's not any other default service 131 | 132 | try: 133 | current = cls.lookup(service_type) 134 | if current.is_default: 135 | raise custom_exceptions.DefaultServiceAlreadyExistException("Default service already exists for type '%s'" % service_type) 136 | except custom_exceptions.ServiceNotFoundException: 137 | pass 138 | 139 | setup_func = meta.get('_setup', None) 140 | if setup_func != None: 141 | _cls()._setup() 142 | 143 | ServiceFactory.registry.setdefault(service_type, {}) 144 | ServiceFactory.registry[service_type][service_vendor] = _cls 145 | 146 | log.msg("Registered %s for service '%s', vendor '%s' (default: %s)" % (_cls, service_type, service_vendor, is_default)) 147 | 148 | def signature(func): 149 | '''Decorate RPC method result with server's signature. 150 | This decorator can be chained with Deferred or inlineCallbacks, thanks to _sign_generator() hack.''' 151 | 152 | def _sign_generator(iterator): 153 | '''Iterate thru generator object, detects BaseException 154 | and inject signature into exception's value (=result of inner method). 155 | This is black magic because of decorating inlineCallbacks methods. 156 | See returnValue documentation for understanding this: 157 | http://twistedmatrix.com/documents/11.0.0/api/twisted.internet.defer.html#returnValue''' 158 | 159 | for i in iterator: 160 | try: 161 | iterator.send((yield i)) 162 | except BaseException as exc: 163 | exc.value = wrap_result_object(exc.value) 164 | exc.value.sign = True 165 | raise 166 | 167 | def _sign_deferred(res): 168 | obj = wrap_result_object(res) 169 | obj.sign = True 170 | return obj 171 | 172 | def _sign_failure(fail): 173 | fail.value = wrap_result_object(fail.value) 174 | fail.value.sign = True 175 | return fail 176 | 177 | def inner(*args, **kwargs): 178 | ret = defer.maybeDeferred(func, *args, **kwargs) 179 | #if isinstance(ret, defer.Deferred): 180 | ret.addCallback(_sign_deferred) 181 | ret.addErrback(_sign_failure) 182 | return ret 183 | # return ret 184 | #elif isinstance(ret, types.GeneratorType): 185 | # return _sign_generator(ret) 186 | #else: 187 | # ret = wrap_result_object(ret) 188 | # ret.sign = True 189 | # return ret 190 | return inner 191 | 192 | def synchronous(func): 193 | '''Run given method synchronously in separate thread and return the result.''' 194 | def inner(*args, **kwargs): 195 | return threads.deferToThread(func, *args, **kwargs) 196 | return inner 197 | 198 | def admin(func): 199 | '''Requires an extra first parameter with superadministrator password''' 200 | import settings 201 | def inner(*args, **kwargs): 202 | if not len(args): 203 | raise custom_exceptions.UnauthorizedException("Missing password") 204 | 205 | if settings.ADMIN_RESTRICT_INTERFACE != None: 206 | ip = args[0].connection_ref()._get_ip() 207 | if settings.ADMIN_RESTRICT_INTERFACE != ip: 208 | raise custom_exceptions.UnauthorizedException("RPC call not allowed from your IP") 209 | 210 | if not settings.ADMIN_PASSWORD_SHA256: 211 | raise custom_exceptions.UnauthorizedException("Admin password not set, RPC call disabled") 212 | 213 | (password, args) = (args[1], [args[0],] + list(args[2:])) 214 | 215 | if hashlib.sha256(password).hexdigest() != settings.ADMIN_PASSWORD_SHA256: 216 | raise custom_exceptions.UnauthorizedException("Wrong password") 217 | 218 | return func(*args, **kwargs) 219 | return inner 220 | 221 | class ServiceMetaclass(type): 222 | def __init__(cls, name, bases, _dict): 223 | super(ServiceMetaclass, cls).__init__(name, bases, _dict) 224 | ServiceFactory.register_service(cls, _dict) 225 | 226 | class GenericService(object): 227 | __metaclass__ = ServiceMetaclass 228 | service_type = None 229 | service_vendor = None 230 | is_default = None 231 | 232 | # Keep weak reference to connection which asked for current 233 | # RPC call. Useful for pubsub mechanism, but use it with care. 234 | # It does not need to point to actual and valid data, so 235 | # you have to check if connection still exists every time. 236 | connection_ref = None 237 | 238 | class ServiceDiscovery(GenericService): 239 | service_type = 'discovery' 240 | service_vendor = 'Stratum' 241 | is_default = True 242 | 243 | def list_services(self): 244 | return ServiceFactory.registry.keys() 245 | 246 | def list_vendors(self, service_type): 247 | return ServiceFactory.registry[service_type].keys() 248 | 249 | def list_methods(self, service_name): 250 | # Accepts also vendors in square brackets: firstbits[firstbits.com] 251 | 252 | # Parse service type and vendor. We don't care about the method name, 253 | # but _split_method needs full path to some RPC method. 254 | (service_type, vendor, _) = ServiceFactory._split_method("%s.foo" % service_name) 255 | service = ServiceFactory.lookup(service_type, vendor) 256 | out = [] 257 | 258 | for name, obj in service.__dict__.items(): 259 | 260 | if name.startswith('_'): 261 | continue 262 | 263 | if not callable(obj): 264 | continue 265 | 266 | out.append(name) 267 | 268 | return out 269 | 270 | def list_params(self, method): 271 | (service_type, vendor, meth) = ServiceFactory._split_method(method) 272 | service = ServiceFactory.lookup(service_type, vendor) 273 | 274 | # Load params and helper text from method attributes 275 | func = service.__dict__[meth] 276 | params = getattr(func, 'params', None) 277 | help_text = getattr(func, 'help_text', None) 278 | 279 | return (help_text, params) 280 | list_params.help_text = "Accepts name of method and returns its description and available parameters. Example: 'firstbits.resolve'" 281 | list_params.params = [('method', 'string', 'Method to lookup for description and parameters.'),] -------------------------------------------------------------------------------- /stratum/protocol.py: -------------------------------------------------------------------------------- 1 | import json 2 | #import jsonical 3 | import time 4 | import socket 5 | 6 | from twisted.protocols.basic import LineOnlyReceiver 7 | from twisted.internet import defer, reactor, error 8 | from twisted.python.failure import Failure 9 | 10 | #import services 11 | import stats 12 | import signature 13 | import custom_exceptions 14 | import connection_registry 15 | import settings 16 | 17 | import logger 18 | log = logger.get_logger('protocol') 19 | 20 | class RequestCounter(object): 21 | def __init__(self): 22 | self.on_finish = defer.Deferred() 23 | self.counter = 0 24 | 25 | def set_count(self, cnt): 26 | self.counter = cnt 27 | 28 | def decrease(self): 29 | self.counter -= 1 30 | if self.counter <= 0: 31 | self.finish() 32 | 33 | def finish(self): 34 | if not self.on_finish.called: 35 | self.on_finish.callback(True) 36 | 37 | class Protocol(LineOnlyReceiver): 38 | delimiter = '\n' 39 | 40 | def _get_id(self): 41 | self.request_id += 1 42 | return self.request_id 43 | 44 | def _get_ip(self): 45 | return self.proxied_ip or self.transport.getPeer().host 46 | 47 | def get_ident(self): 48 | # Get global unique ID of connection 49 | return "%s:%s" % (self.proxied_ip or self.transport.getPeer().host, "%x" % id(self)) 50 | 51 | def get_session(self): 52 | return self.session 53 | 54 | def connectionMade(self): 55 | try: 56 | self.transport.setTcpNoDelay(True) 57 | self.transport.setTcpKeepAlive(True) 58 | self.transport.socket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120) # Seconds before sending keepalive probes 59 | self.transport.socket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 1) # Interval in seconds between keepalive probes 60 | self.transport.socket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5) # Failed keepalive probles before declaring other end dead 61 | except: 62 | # Supported only by the socket transport, 63 | # but there's really no better place in code to trigger this. 64 | pass 65 | 66 | # Read settings.TCP_PROXY_PROTOCOL documentation 67 | self.expect_tcp_proxy_protocol_header = self.factory.__dict__.get('tcp_proxy_protocol_enable', False) 68 | self.proxied_ip = None # IP obtained from TCP proxy protocol 69 | 70 | self.request_id = 0 71 | self.lookup_table = {} 72 | self.event_handler = self.factory.event_handler() 73 | self.on_disconnect = defer.Deferred() 74 | self.on_finish = None # Will point to defer which is called 75 | # once all client requests are processed 76 | 77 | # Initiate connection session 78 | self.session = {} 79 | 80 | stats.PeerStats.client_connected(self._get_ip()) 81 | log.debug("Connected %s" % self.transport.getPeer().host) 82 | connection_registry.ConnectionRegistry.add_connection(self) 83 | 84 | def transport_write(self, data): 85 | '''Overwrite this if transport needs some extra care about data written 86 | to the socket, like adding message format in websocket.''' 87 | try: 88 | self.transport.write(data) 89 | except AttributeError: 90 | # Transport is disconnected 91 | pass 92 | 93 | def connectionLost(self, reason): 94 | if self.on_disconnect != None and not self.on_disconnect.called: 95 | self.on_disconnect.callback(self) 96 | self.on_disconnect = None 97 | 98 | stats.PeerStats.client_disconnected(self._get_ip()) 99 | connection_registry.ConnectionRegistry.remove_connection(self) 100 | self.transport = None # Fixes memory leak (cyclic reference) 101 | 102 | def writeJsonRequest(self, method, params, is_notification=False): 103 | request_id = None if is_notification else self._get_id() 104 | serialized = json.dumps({'id': request_id, 'method': method, 'params': params}) 105 | 106 | if self.factory.debug: 107 | log.debug("< %s" % serialized) 108 | 109 | self.transport_write("%s\n" % serialized) 110 | return request_id 111 | 112 | def writeJsonResponse(self, data, message_id, use_signature=False, sign_method='', sign_params=[]): 113 | if use_signature: 114 | serialized = signature.jsonrpc_dumps_sign(self.factory.signing_key, self.factory.signing_id, False,\ 115 | message_id, sign_method, sign_params, data, None) 116 | else: 117 | serialized = json.dumps({'id': message_id, 'result': data, 'error': None}) 118 | 119 | if self.factory.debug: 120 | log.debug("< %s" % serialized) 121 | 122 | self.transport_write("%s\n" % serialized) 123 | 124 | def writeJsonError(self, code, message, traceback, message_id, use_signature=False, sign_method='', sign_params=[]): 125 | if use_signature: 126 | serialized = signature.jsonrpc_dumps_sign(self.factory.signing_key, self.factory.signing_id, False,\ 127 | message_id, sign_method, sign_params, None, (code, message, traceback)) 128 | else: 129 | serialized = json.dumps({'id': message_id, 'result': None, 'error': (code, message, traceback)}) 130 | 131 | self.transport_write("%s\n" % serialized) 132 | 133 | def writeGeneralError(self, message, code=-1): 134 | log.error(message) 135 | return self.writeJsonError(code, message, None, None) 136 | 137 | def process_response(self, data, message_id, sign_method, sign_params, request_counter): 138 | self.writeJsonResponse(data.result, message_id, data.sign, sign_method, sign_params) 139 | request_counter.decrease() 140 | 141 | 142 | def process_failure(self, failure, message_id, sign_method, sign_params, request_counter): 143 | if not isinstance(failure.value, custom_exceptions.ServiceException): 144 | # All handled exceptions should inherit from ServiceException class. 145 | # Throwing other exception class means that it is unhandled error 146 | # and we should log it. 147 | log.exception(failure) 148 | 149 | sign = False 150 | code = getattr(failure.value, 'code', -1) 151 | 152 | #if isinstance(failure.value, services.ResultObject): 153 | # # Strip ResultObject 154 | # sign = failure.value.sign 155 | # failure.value = failure.value.result 156 | 157 | if message_id != None: 158 | # Other party doesn't care of error state for notifications 159 | if settings.DEBUG: 160 | tb = failure.getBriefTraceback() 161 | else: 162 | tb = None 163 | self.writeJsonError(code, failure.getErrorMessage(), tb, message_id, sign, sign_method, sign_params) 164 | 165 | request_counter.decrease() 166 | 167 | def dataReceived(self, data, request_counter=None): 168 | '''Original code from Twisted, hacked for request_counter proxying. 169 | request_counter is hack for HTTP transport, didn't found cleaner solution how 170 | to indicate end of request processing in asynchronous manner. 171 | 172 | TODO: This would deserve some unit test to be sure that future twisted versions 173 | will work nicely with this.''' 174 | 175 | if request_counter == None: 176 | request_counter = RequestCounter() 177 | 178 | lines = (self._buffer+data).split(self.delimiter) 179 | self._buffer = lines.pop(-1) 180 | request_counter.set_count(len(lines)) 181 | self.on_finish = request_counter.on_finish 182 | 183 | for line in lines: 184 | if self.transport.disconnecting: 185 | request_counter.finish() 186 | return 187 | if len(line) > self.MAX_LENGTH: 188 | request_counter.finish() 189 | return self.lineLengthExceeded(line) 190 | else: 191 | try: 192 | self.lineReceived(line, request_counter) 193 | except Exception as exc: 194 | request_counter.finish() 195 | #log.exception("Processing of message failed") 196 | log.warning("Failed message: %s from %s" % (str(exc), self._get_ip())) 197 | return error.ConnectionLost('Processing of message failed') 198 | 199 | if len(self._buffer) > self.MAX_LENGTH: 200 | request_counter.finish() 201 | return self.lineLengthExceeded(self._buffer) 202 | 203 | def lineReceived(self, line, request_counter): 204 | if self.expect_tcp_proxy_protocol_header: 205 | # This flag may be set only for TCP transport AND when TCP_PROXY_PROTOCOL 206 | # is enabled in server config. Then we expect the first line of the stream 207 | # may contain proxy metadata. 208 | 209 | # We don't expect this header during this session anymore 210 | self.expect_tcp_proxy_protocol_header = False 211 | 212 | if line.startswith('PROXY'): 213 | self.proxied_ip = line.split()[2] 214 | 215 | # Let's process next line 216 | request_counter.decrease() 217 | return 218 | 219 | try: 220 | message = json.loads(line) 221 | except: 222 | #self.writeGeneralError("Cannot decode message '%s'" % line) 223 | request_counter.finish() 224 | raise custom_exceptions.ProtocolException("Cannot decode message '%s'" % line.strip()) 225 | 226 | if self.factory.debug: 227 | log.debug("> %s" % message) 228 | 229 | msg_id = message.get('id', 0) 230 | msg_method = message.get('method') 231 | msg_params = message.get('params') 232 | msg_result = message.get('result') 233 | msg_error = message.get('error') 234 | 235 | if msg_method: 236 | # It's a RPC call or notification 237 | try: 238 | result = self.event_handler._handle_event(msg_method, msg_params, connection_ref=self) 239 | if result == None and msg_id != None: 240 | # event handler must return Deferred object or raise an exception for RPC request 241 | raise custom_exceptions.MethodNotFoundException("Event handler cannot process method '%s'" % msg_method) 242 | except: 243 | failure = Failure() 244 | self.process_failure(failure, msg_id, msg_method, msg_params, request_counter) 245 | 246 | else: 247 | if msg_id == None: 248 | # It's notification, don't expect the response 249 | request_counter.decrease() 250 | else: 251 | # It's a RPC call 252 | result.addCallback(self.process_response, msg_id, msg_method, msg_params, request_counter) 253 | result.addErrback(self.process_failure, msg_id, msg_method, msg_params, request_counter) 254 | 255 | elif msg_id: 256 | # It's a RPC response 257 | # Perform lookup to the table of waiting requests. 258 | request_counter.decrease() 259 | 260 | try: 261 | meta = self.lookup_table[msg_id] 262 | del self.lookup_table[msg_id] 263 | except KeyError: 264 | # When deferred object for given message ID isn't found, it's an error 265 | raise custom_exceptions.ProtocolException("Lookup for deferred object for message ID '%s' failed." % msg_id) 266 | 267 | # If there's an error, handle it as errback 268 | # If both result and error are null, handle it as a success with blank result 269 | if msg_error != None: 270 | meta['defer'].errback(custom_exceptions.RemoteServiceException(msg_error[0], msg_error[1], msg_error[2])) 271 | else: 272 | meta['defer'].callback(msg_result) 273 | 274 | else: 275 | request_counter.decrease() 276 | raise custom_exceptions.ProtocolException("Cannot handle message '%s'" % line) 277 | 278 | def rpc(self, method, params, is_notification=False): 279 | ''' 280 | This method performs remote RPC call. 281 | 282 | If method should expect an response, it store 283 | request ID to lookup table and wait for corresponding 284 | response message. 285 | ''' 286 | 287 | request_id = self.writeJsonRequest(method, params, is_notification) 288 | 289 | if is_notification: 290 | return 291 | 292 | d = defer.Deferred() 293 | self.lookup_table[request_id] = {'defer': d, 'method': method, 'params': params} 294 | return d 295 | 296 | class ClientProtocol(Protocol): 297 | def connectionMade(self): 298 | Protocol.connectionMade(self) 299 | self.factory.client = self 300 | 301 | if self.factory.timeout_handler: 302 | self.factory.timeout_handler.cancel() 303 | self.factory.timeout_handler = None 304 | 305 | if isinstance(getattr(self.factory, 'after_connect', None), list): 306 | log.debug("Resuming connection: %s" % self.factory.after_connect) 307 | for cmd in self.factory.after_connect: 308 | self.rpc(cmd[0], cmd[1]) 309 | 310 | if not self.factory.on_connect.called: 311 | d = self.factory.on_connect 312 | self.factory.on_connect = defer.Deferred() 313 | d.callback(self.factory) 314 | 315 | 316 | #d = self.rpc('node.get_peers', []) 317 | #d.addCallback(self.factory.add_peers) 318 | 319 | def connectionLost(self, reason): 320 | self.factory.client = None 321 | 322 | if self.factory.timeout_handler: 323 | self.factory.timeout_handler.cancel() 324 | self.factory.timeout_handler = None 325 | 326 | if not self.factory.on_disconnect.called: 327 | d = self.factory.on_disconnect 328 | self.factory.on_disconnect = defer.Deferred() 329 | d.callback(self.factory) 330 | 331 | Protocol.connectionLost(self, reason) 332 | -------------------------------------------------------------------------------- /distribute_setup.py: -------------------------------------------------------------------------------- 1 | #!python 2 | """Bootstrap distribute installation 3 | 4 | If you want to use setuptools in your package's setup.py, just include this 5 | file in the same directory with it, and add this to the top of your setup.py:: 6 | 7 | from distribute_setup import use_setuptools 8 | use_setuptools() 9 | 10 | If you want to require a specific version of setuptools, set a download 11 | mirror, or use an alternate download directory, you can do so by supplying 12 | the appropriate options to ``use_setuptools()``. 13 | 14 | This file can also be run as a script to install or upgrade setuptools. 15 | """ 16 | import os 17 | import sys 18 | import time 19 | import fnmatch 20 | import tempfile 21 | import tarfile 22 | from distutils import log 23 | 24 | try: 25 | from site import USER_SITE 26 | except ImportError: 27 | USER_SITE = None 28 | 29 | try: 30 | import subprocess 31 | 32 | def _python_cmd(*args): 33 | args = (sys.executable,) + args 34 | return subprocess.call(args) == 0 35 | 36 | except ImportError: 37 | # will be used for python 2.3 38 | def _python_cmd(*args): 39 | args = (sys.executable,) + args 40 | # quoting arguments if windows 41 | if sys.platform == 'win32': 42 | def quote(arg): 43 | if ' ' in arg: 44 | return '"%s"' % arg 45 | return arg 46 | args = [quote(arg) for arg in args] 47 | return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 48 | 49 | DEFAULT_VERSION = "0.6.28" 50 | DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" 51 | SETUPTOOLS_FAKED_VERSION = "0.6c11" 52 | 53 | SETUPTOOLS_PKG_INFO = """\ 54 | Metadata-Version: 1.0 55 | Name: setuptools 56 | Version: %s 57 | Summary: xxxx 58 | Home-page: xxx 59 | Author: xxx 60 | Author-email: xxx 61 | License: xxx 62 | Description: xxx 63 | """ % SETUPTOOLS_FAKED_VERSION 64 | 65 | 66 | def _install(tarball, install_args=()): 67 | # extracting the tarball 68 | tmpdir = tempfile.mkdtemp() 69 | log.warn('Extracting in %s', tmpdir) 70 | old_wd = os.getcwd() 71 | try: 72 | os.chdir(tmpdir) 73 | tar = tarfile.open(tarball) 74 | _extractall(tar) 75 | tar.close() 76 | 77 | # going in the directory 78 | subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 79 | os.chdir(subdir) 80 | log.warn('Now working in %s', subdir) 81 | 82 | # installing 83 | log.warn('Installing Distribute') 84 | if not _python_cmd('setup.py', 'install', *install_args): 85 | log.warn('Something went wrong during the installation.') 86 | log.warn('See the error message above.') 87 | finally: 88 | os.chdir(old_wd) 89 | 90 | 91 | def _build_egg(egg, tarball, to_dir): 92 | # extracting the tarball 93 | tmpdir = tempfile.mkdtemp() 94 | log.warn('Extracting in %s', tmpdir) 95 | old_wd = os.getcwd() 96 | try: 97 | os.chdir(tmpdir) 98 | tar = tarfile.open(tarball) 99 | _extractall(tar) 100 | tar.close() 101 | 102 | # going in the directory 103 | subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 104 | os.chdir(subdir) 105 | log.warn('Now working in %s', subdir) 106 | 107 | # building an egg 108 | log.warn('Building a Distribute egg in %s', to_dir) 109 | _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) 110 | 111 | finally: 112 | os.chdir(old_wd) 113 | # returning the result 114 | log.warn(egg) 115 | if not os.path.exists(egg): 116 | raise IOError('Could not build the egg.') 117 | 118 | 119 | def _do_download(version, download_base, to_dir, download_delay): 120 | egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg' 121 | % (version, sys.version_info[0], sys.version_info[1])) 122 | if not os.path.exists(egg): 123 | tarball = download_setuptools(version, download_base, 124 | to_dir, download_delay) 125 | _build_egg(egg, tarball, to_dir) 126 | sys.path.insert(0, egg) 127 | import setuptools 128 | setuptools.bootstrap_install_from = egg 129 | 130 | 131 | def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, 132 | to_dir=os.curdir, download_delay=15, no_fake=True): 133 | # making sure we use the absolute path 134 | to_dir = os.path.abspath(to_dir) 135 | was_imported = 'pkg_resources' in sys.modules or \ 136 | 'setuptools' in sys.modules 137 | try: 138 | try: 139 | import pkg_resources 140 | if not hasattr(pkg_resources, '_distribute'): 141 | if not no_fake: 142 | _fake_setuptools() 143 | raise ImportError 144 | except ImportError: 145 | return _do_download(version, download_base, to_dir, download_delay) 146 | try: 147 | pkg_resources.require("distribute>=" + version) 148 | return 149 | except pkg_resources.VersionConflict: 150 | e = sys.exc_info()[1] 151 | if was_imported: 152 | sys.stderr.write( 153 | "The required version of distribute (>=%s) is not available,\n" 154 | "and can't be installed while this script is running. Please\n" 155 | "install a more recent version first, using\n" 156 | "'easy_install -U distribute'." 157 | "\n\n(Currently using %r)\n" % (version, e.args[0])) 158 | sys.exit(2) 159 | else: 160 | del pkg_resources, sys.modules['pkg_resources'] # reload ok 161 | return _do_download(version, download_base, to_dir, 162 | download_delay) 163 | except pkg_resources.DistributionNotFound: 164 | return _do_download(version, download_base, to_dir, 165 | download_delay) 166 | finally: 167 | if not no_fake: 168 | _create_fake_setuptools_pkg_info(to_dir) 169 | 170 | 171 | def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, 172 | to_dir=os.curdir, delay=15): 173 | """Download distribute from a specified location and return its filename 174 | 175 | `version` should be a valid distribute version number that is available 176 | as an egg for download under the `download_base` URL (which should end 177 | with a '/'). `to_dir` is the directory where the egg will be downloaded. 178 | `delay` is the number of seconds to pause before an actual download 179 | attempt. 180 | """ 181 | # making sure we use the absolute path 182 | to_dir = os.path.abspath(to_dir) 183 | try: 184 | from urllib.request import urlopen 185 | except ImportError: 186 | from urllib2 import urlopen 187 | tgz_name = "distribute-%s.tar.gz" % version 188 | url = download_base + tgz_name 189 | saveto = os.path.join(to_dir, tgz_name) 190 | src = dst = None 191 | if not os.path.exists(saveto): # Avoid repeated downloads 192 | try: 193 | log.warn("Downloading %s", url) 194 | src = urlopen(url) 195 | # Read/write all in one block, so we don't create a corrupt file 196 | # if the download is interrupted. 197 | data = src.read() 198 | dst = open(saveto, "wb") 199 | dst.write(data) 200 | finally: 201 | if src: 202 | src.close() 203 | if dst: 204 | dst.close() 205 | return os.path.realpath(saveto) 206 | 207 | 208 | def _no_sandbox(function): 209 | def __no_sandbox(*args, **kw): 210 | try: 211 | from setuptools.sandbox import DirectorySandbox 212 | if not hasattr(DirectorySandbox, '_old'): 213 | def violation(*args): 214 | pass 215 | DirectorySandbox._old = DirectorySandbox._violation 216 | DirectorySandbox._violation = violation 217 | patched = True 218 | else: 219 | patched = False 220 | except ImportError: 221 | patched = False 222 | 223 | try: 224 | return function(*args, **kw) 225 | finally: 226 | if patched: 227 | DirectorySandbox._violation = DirectorySandbox._old 228 | del DirectorySandbox._old 229 | 230 | return __no_sandbox 231 | 232 | 233 | def _patch_file(path, content): 234 | """Will backup the file then patch it""" 235 | existing_content = open(path).read() 236 | if existing_content == content: 237 | # already patched 238 | log.warn('Already patched.') 239 | return False 240 | log.warn('Patching...') 241 | _rename_path(path) 242 | f = open(path, 'w') 243 | try: 244 | f.write(content) 245 | finally: 246 | f.close() 247 | return True 248 | 249 | _patch_file = _no_sandbox(_patch_file) 250 | 251 | 252 | def _same_content(path, content): 253 | return open(path).read() == content 254 | 255 | 256 | def _rename_path(path): 257 | new_name = path + '.OLD.%s' % time.time() 258 | log.warn('Renaming %s into %s', path, new_name) 259 | os.rename(path, new_name) 260 | return new_name 261 | 262 | 263 | def _remove_flat_installation(placeholder): 264 | if not os.path.isdir(placeholder): 265 | log.warn('Unkown installation at %s', placeholder) 266 | return False 267 | found = False 268 | for file in os.listdir(placeholder): 269 | if fnmatch.fnmatch(file, 'setuptools*.egg-info'): 270 | found = True 271 | break 272 | if not found: 273 | log.warn('Could not locate setuptools*.egg-info') 274 | return 275 | 276 | log.warn('Removing elements out of the way...') 277 | pkg_info = os.path.join(placeholder, file) 278 | if os.path.isdir(pkg_info): 279 | patched = _patch_egg_dir(pkg_info) 280 | else: 281 | patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) 282 | 283 | if not patched: 284 | log.warn('%s already patched.', pkg_info) 285 | return False 286 | # now let's move the files out of the way 287 | for element in ('setuptools', 'pkg_resources.py', 'site.py'): 288 | element = os.path.join(placeholder, element) 289 | if os.path.exists(element): 290 | _rename_path(element) 291 | else: 292 | log.warn('Could not find the %s element of the ' 293 | 'Setuptools distribution', element) 294 | return True 295 | 296 | _remove_flat_installation = _no_sandbox(_remove_flat_installation) 297 | 298 | 299 | def _after_install(dist): 300 | log.warn('After install bootstrap.') 301 | placeholder = dist.get_command_obj('install').install_purelib 302 | _create_fake_setuptools_pkg_info(placeholder) 303 | 304 | 305 | def _create_fake_setuptools_pkg_info(placeholder): 306 | if not placeholder or not os.path.exists(placeholder): 307 | log.warn('Could not find the install location') 308 | return 309 | pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) 310 | setuptools_file = 'setuptools-%s-py%s.egg-info' % \ 311 | (SETUPTOOLS_FAKED_VERSION, pyver) 312 | pkg_info = os.path.join(placeholder, setuptools_file) 313 | if os.path.exists(pkg_info): 314 | log.warn('%s already exists', pkg_info) 315 | return 316 | 317 | if not os.access(pkg_info, os.W_OK): 318 | log.warn("Don't have permissions to write %s, skipping", pkg_info) 319 | 320 | log.warn('Creating %s', pkg_info) 321 | f = open(pkg_info, 'w') 322 | try: 323 | f.write(SETUPTOOLS_PKG_INFO) 324 | finally: 325 | f.close() 326 | 327 | pth_file = os.path.join(placeholder, 'setuptools.pth') 328 | log.warn('Creating %s', pth_file) 329 | f = open(pth_file, 'w') 330 | try: 331 | f.write(os.path.join(os.curdir, setuptools_file)) 332 | finally: 333 | f.close() 334 | 335 | _create_fake_setuptools_pkg_info = _no_sandbox( 336 | _create_fake_setuptools_pkg_info 337 | ) 338 | 339 | 340 | def _patch_egg_dir(path): 341 | # let's check if it's already patched 342 | pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') 343 | if os.path.exists(pkg_info): 344 | if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): 345 | log.warn('%s already patched.', pkg_info) 346 | return False 347 | _rename_path(path) 348 | os.mkdir(path) 349 | os.mkdir(os.path.join(path, 'EGG-INFO')) 350 | pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') 351 | f = open(pkg_info, 'w') 352 | try: 353 | f.write(SETUPTOOLS_PKG_INFO) 354 | finally: 355 | f.close() 356 | return True 357 | 358 | _patch_egg_dir = _no_sandbox(_patch_egg_dir) 359 | 360 | 361 | def _before_install(): 362 | log.warn('Before install bootstrap.') 363 | _fake_setuptools() 364 | 365 | 366 | def _under_prefix(location): 367 | if 'install' not in sys.argv: 368 | return True 369 | args = sys.argv[sys.argv.index('install') + 1:] 370 | for index, arg in enumerate(args): 371 | for option in ('--root', '--prefix'): 372 | if arg.startswith('%s=' % option): 373 | top_dir = arg.split('root=')[-1] 374 | return location.startswith(top_dir) 375 | elif arg == option: 376 | if len(args) > index: 377 | top_dir = args[index + 1] 378 | return location.startswith(top_dir) 379 | if arg == '--user' and USER_SITE is not None: 380 | return location.startswith(USER_SITE) 381 | return True 382 | 383 | 384 | def _fake_setuptools(): 385 | log.warn('Scanning installed packages') 386 | try: 387 | import pkg_resources 388 | except ImportError: 389 | # we're cool 390 | log.warn('Setuptools or Distribute does not seem to be installed.') 391 | return 392 | ws = pkg_resources.working_set 393 | try: 394 | setuptools_dist = ws.find( 395 | pkg_resources.Requirement.parse('setuptools', replacement=False) 396 | ) 397 | except TypeError: 398 | # old distribute API 399 | setuptools_dist = ws.find( 400 | pkg_resources.Requirement.parse('setuptools') 401 | ) 402 | 403 | if setuptools_dist is None: 404 | log.warn('No setuptools distribution found') 405 | return 406 | # detecting if it was already faked 407 | setuptools_location = setuptools_dist.location 408 | log.warn('Setuptools installation detected at %s', setuptools_location) 409 | 410 | # if --root or --preix was provided, and if 411 | # setuptools is not located in them, we don't patch it 412 | if not _under_prefix(setuptools_location): 413 | log.warn('Not patching, --root or --prefix is installing Distribute' 414 | ' in another location') 415 | return 416 | 417 | # let's see if its an egg 418 | if not setuptools_location.endswith('.egg'): 419 | log.warn('Non-egg installation') 420 | res = _remove_flat_installation(setuptools_location) 421 | if not res: 422 | return 423 | else: 424 | log.warn('Egg installation') 425 | pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') 426 | if (os.path.exists(pkg_info) and 427 | _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): 428 | log.warn('Already patched.') 429 | return 430 | log.warn('Patching...') 431 | # let's create a fake egg replacing setuptools one 432 | res = _patch_egg_dir(setuptools_location) 433 | if not res: 434 | return 435 | log.warn('Patched done.') 436 | _relaunch() 437 | 438 | 439 | def _relaunch(): 440 | log.warn('Relaunching...') 441 | # we have to relaunch the process 442 | # pip marker to avoid a relaunch bug 443 | _cmd = ['-c', 'install', '--single-version-externally-managed'] 444 | if sys.argv[:3] == _cmd: 445 | sys.argv[0] = 'setup.py' 446 | args = [sys.executable] + sys.argv 447 | sys.exit(subprocess.call(args)) 448 | 449 | 450 | def _extractall(self, path=".", members=None): 451 | """Extract all members from the archive to the current working 452 | directory and set owner, modification time and permissions on 453 | directories afterwards. `path' specifies a different directory 454 | to extract to. `members' is optional and must be a subset of the 455 | list returned by getmembers(). 456 | """ 457 | import copy 458 | import operator 459 | from tarfile import ExtractError 460 | directories = [] 461 | 462 | if members is None: 463 | members = self 464 | 465 | for tarinfo in members: 466 | if tarinfo.isdir(): 467 | # Extract directories with a safe mode. 468 | directories.append(tarinfo) 469 | tarinfo = copy.copy(tarinfo) 470 | tarinfo.mode = 448 # decimal for oct 0700 471 | self.extract(tarinfo, path) 472 | 473 | # Reverse sort directories. 474 | if sys.version_info < (2, 4): 475 | def sorter(dir1, dir2): 476 | return cmp(dir1.name, dir2.name) 477 | directories.sort(sorter) 478 | directories.reverse() 479 | else: 480 | directories.sort(key=operator.attrgetter('name'), reverse=True) 481 | 482 | # Set correct owner, mtime and filemode on directories. 483 | for tarinfo in directories: 484 | dirpath = os.path.join(path, tarinfo.name) 485 | try: 486 | self.chown(tarinfo, dirpath) 487 | self.utime(tarinfo, dirpath) 488 | self.chmod(tarinfo, dirpath) 489 | except ExtractError: 490 | e = sys.exc_info()[1] 491 | if self.errorlevel > 1: 492 | raise 493 | else: 494 | self._dbg(1, "tarfile: %s" % e) 495 | 496 | 497 | def _build_install_args(argv): 498 | install_args = [] 499 | user_install = '--user' in argv 500 | if user_install and sys.version_info < (2, 6): 501 | log.warn("--user requires Python 2.6 or later") 502 | raise SystemExit(1) 503 | if user_install: 504 | install_args.append('--user') 505 | return install_args 506 | 507 | 508 | def main(argv, version=DEFAULT_VERSION): 509 | """Install or upgrade setuptools and EasyInstall""" 510 | tarball = download_setuptools() 511 | _install(tarball, _build_install_args(argv)) 512 | 513 | 514 | if __name__ == '__main__': 515 | main(sys.argv[1:]) 516 | --------------------------------------------------------------------------------