├── .dockerignore ├── .gitignore ├── Dockerfile ├── README.rst └── app ├── .flake8 ├── app.py ├── bot ├── __init__.py ├── client.py ├── commands.py ├── helpers.py ├── signals.py └── views.py ├── chunks.py ├── config ├── __init__.py ├── prod.py └── testing.py ├── conftest.py ├── customerio ├── __init__.py ├── client.py ├── conftest.py ├── const.py ├── geo.py ├── signals.py ├── test_customerio.py └── test_signals.py ├── errors.py ├── eth ├── batch.py ├── models.py └── views.py ├── idm ├── __init__.py ├── const.py ├── errors.py ├── helpers.py ├── models.py ├── verify.py └── views.py ├── ids ├── const.py ├── models.py ├── test_ids.py └── views.py ├── onfid ├── __init__.py ├── api.py ├── const.py ├── geo.py ├── helpers.py ├── models.py ├── signals.py └── views.py ├── pylint.cfg ├── requirements.txt ├── routes.py ├── schema.py ├── sentry.py ├── session.py ├── signals.py ├── start.sh ├── tokens ├── __init__.py └── test_token.py ├── upload ├── errors.py ├── models.py ├── s3.py ├── test_upload.py └── views.py ├── user ├── __init__.py ├── auth.py ├── errors.py ├── models.py ├── state.py ├── test_auth.py ├── test_user_info.py ├── verifications.py └── views.py ├── uwsgi.ini └── wsgi.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | app/__pycache__/ 3 | app/.cache/ 4 | libs/ 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # Jupyter Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # SageMath parsed files 79 | *.sage.py 80 | 81 | # Environments 82 | .env 83 | .venv 84 | env/ 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | .spyproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # mkdocs documentation 96 | /site 97 | 98 | # mypy 99 | .mypy_cache/ 100 | 101 | # Vim 102 | *.swp 103 | 104 | # Application data 105 | #models 106 | data 107 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | EXPOSE 80 3 | WORKDIR /app 4 | CMD ["/app/start.sh"] 5 | 6 | # Install magicwand 7 | ADD app/requirements.txt /tmp/requirements.txt 8 | RUN pip install -r /tmp/requirements.txt 9 | 10 | ADD app /app 11 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | !!! Important !!! 2 | ----------------- 3 | 4 | This is a snapshot of our post-whitelist state of the service that handled whitelist. Tests are partially outdated 5 | (as well as some bits of documentation). 6 | 7 | Code here is FYI only. It is not in use any more and all data related to the whitelist is now wiped from S3 and our 8 | servers. 9 | 10 | The flow 11 | -------- 12 | 13 | 1. Obtain device fingerprint from Augur 14 | 2. Submit KYC data ``POST /v1/user {....}`` -> ``{"state": "", "token": "", ....}`` 15 | 3. Call ``GET /v1/user`` to check user's state. You're waiting for ``state: "info_approved"`` 16 | 4. Get link for image1: ``POST /v1/upload {"filename": "...", "content_type": "...", "size": ...}`` -> ``{"put_url": "...", "id": "..."}`` 17 | 5. Repeat for second image 18 | 6. Upload images to S3 19 | 7. Create ID package: ``POST /v1/ids {"upload1": "", ...}`` -> ``{"id": "...", ...}`` 20 | 8. Submit package to verification: ``POST /v1/ids//verify {}`` -> get user status 21 | 9. That's it. From now on it's only checking user's status 22 | 23 | Heaviest routes for us are ``/v1/ids`` and ``/v1/ids//verify`` as they pull data from s3 and submit it to IDM 24 | 25 | Example response for user data:: 26 | 27 | { 28 | "address": { 29 | "address_components": [ 30 | { 31 | "long_name": "Ashford Dunwoody Road Northeast", 32 | "short_name": "Ashford Dunwoody Rd NE", 33 | "types": [ 34 | "route" 35 | ] 36 | }, 37 | { 38 | "long_name": "Dunwoody", 39 | "short_name": "Dunwoody", 40 | "types": [ 41 | "locality", 42 | "political" 43 | ] 44 | }, 45 | { 46 | "long_name": "DeKalb County", 47 | "short_name": "Dekalb County", 48 | "types": [ 49 | "administrative_area_level_2", 50 | "political" 51 | ] 52 | }, 53 | { 54 | "long_name": "Georgia", 55 | "short_name": "GA", 56 | "types": [ 57 | "administrative_area_level_1", 58 | "political" 59 | ] 60 | }, 61 | { 62 | "long_name": "United States", 63 | "short_name": "US", 64 | "types": [ 65 | "country", 66 | "political" 67 | ] 68 | } 69 | ], 70 | "description": "Ashford Dunwoody Road Northeast, Dunwoody, GA, United States", 71 | "formatted_address": "Ashford Dunwoody Rd NE, Dunwoody, GA, USA", 72 | "latitude": 33.9302397, 73 | "longitude": -84.3373449, 74 | "name": "Ashford Dunwoody Road Northeast", 75 | "place_id": "EilBc2hmb3JkIER1bndvb2R5IFJkIE5FLCBEdW53b29keSwgR0EsIFVTQQ", 76 | "scope": "GOOGLE", 77 | "types": [ 78 | "route" 79 | ], 80 | "utc_offset": -300 81 | }, 82 | "country_code": "AW", 83 | "dob": 1451862000, 84 | "email": "asdfasdf@example.com", 85 | "eth_address": "1212121212121212121212121212121212121212", 86 | "eth_amount": 123.4, 87 | "eth_cap": null, 88 | "id": "5a74cd021ba9f80001ecfc7e", 89 | "name": "asfdsadf qwdfas", 90 | "phone": "12345678", 91 | "state": "info_verified", 92 | "token": "WyI1YTc0Y2QwMjFiYTlmODAwMDFlY2ZjN2UiXQ.fXvNabow5av7twL4THqQbX0eCps" 93 | } 94 | 95 | ``POST /v1/user`` schema:: 96 | 97 | schema = Schema({ 98 | 'name': All(Length(5, 30), str), 99 | 'email': Email(), 100 | 'dob': Coerce(to_datetime), # unix timestamp 101 | 'address': Coerce(to_addr), 102 | 'country_code': All(Length(2, 3), str), 103 | 'phone': All(Length(8, 20), str), 104 | 'eth_address': Coerce(to_eth), # ^(0x)?[0-9a-fA-F]{40}$ 105 | 'eth_amount': All(Range(0, 100), float), 106 | 'telegram': Coerce(to_telegram), # ^@?(?P[0-9a-z_]{5,25})$ 107 | 'confirmed_location': bool, # Should be True 108 | 'dfp': All(Length(10, 300), str), 109 | }, extra=REMOVE_EXTRA, required=True) 110 | 111 | 112 | ``POST /v1/upload`` schema:: 113 | 114 | schema = Schema({ 115 | 'filename': All(Length(3, 30), str), 116 | 'content_type': All(Length(5, 20), str), # gif or jpeg 117 | 'size': Range(400 * 1024, 4 * 1024 * 1024), # 400KB..4MB 118 | }, extra=REMOVE_EXTRA, required=True) 119 | 120 | ``POST /v1/ids`` schema:: 121 | 122 | schema = Schema({ 123 | 'upload1': Coerce(ObjectId), 124 | 'upload2': Coerce(ObjectId), 125 | 'doc_type': In(DOC_TYPES), # DL, PP, ID, RP, UB 126 | 'doc_country': All(Length(2, 2), str), 127 | Optional('doc_state', default=None): Any(None, All(Length(2, 20), str)), 128 | }, extra=REMOVE_EXTRA, required=True) 129 | 130 | -------------------------------------------------------------------------------- /app/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | 4 | -------------------------------------------------------------------------------- /app/app.py: -------------------------------------------------------------------------------- 1 | import logging.config 2 | 3 | from flask import Flask, jsonify 4 | from flask_cors import CORS 5 | from mongoengine import connect 6 | 7 | import customerio # noqa 8 | from errors import register_errors 9 | from routes import TokenConverter, load_blueprints 10 | from sentry import setup_sentry 11 | 12 | _ = customerio # noqa - make pycharm happy 13 | 14 | 15 | def create_app(environment: str = 'prod') -> Flask: 16 | app = Flask(__name__) 17 | CORS(app, resources={r"/v1/*": {"origins": "*"}}) 18 | 19 | # Load default config files 20 | app.config.from_pyfile(f'config/__init__.py') 21 | app.config.from_pyfile(f'config/{environment}.py') 22 | 23 | # Setup app-wide logging 24 | logging.config.dictConfig(app.config['LOGGING']) 25 | logging.getLogger('root').setLevel(logging.DEBUG if app.config['DEBUG'] else logging.INFO) 26 | logging.getLogger('boto').setLevel(logging.CRITICAL) 27 | 28 | # Setup sentry error logging 29 | if not app.config['TESTING']: 30 | setup_sentry(app) 31 | 32 | # Connect DB 33 | connect('default', host=app.config['MONGODB_URI']) 34 | 35 | app.url_map.converters['token'] = TokenConverter 36 | 37 | register_errors(app) 38 | 39 | for blueprint in load_blueprints(): 40 | app.register_blueprint(blueprint) 41 | 42 | @app.route('/v1/ping') 43 | def ping_route(): 44 | return jsonify({'reply': 'pong'}) 45 | 46 | return app 47 | -------------------------------------------------------------------------------- /app/bot/__init__.py: -------------------------------------------------------------------------------- 1 | import bot.commands 2 | import bot.signals 3 | -------------------------------------------------------------------------------- /app/bot/client.py: -------------------------------------------------------------------------------- 1 | import telegram.ext 2 | from mock import Mock 3 | from telegram import Bot 4 | 5 | from config import TELEGRAM_TOKEN 6 | 7 | if TELEGRAM_TOKEN: 8 | bot = Bot(TELEGRAM_TOKEN) 9 | else: 10 | bot = Mock() 11 | 12 | dispatcher = telegram.ext.Dispatcher(bot, None) 13 | 14 | -------------------------------------------------------------------------------- /app/bot/commands.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from io import StringIO 3 | 4 | import gevent 5 | import telegram 6 | from bson import ObjectId 7 | from mongoengine import Q 8 | from telegram.ext import CommandHandler 9 | 10 | from bot.client import dispatcher 11 | from bot.helpers import restricted 12 | from control import export 13 | from ids.models import IDUpload 14 | from onfid.models import Check 15 | from upload.models import Upload 16 | from upload.s3 import s3 17 | from user.models import User 18 | from user.state import ALL_STATES 19 | from user.views import to_eth 20 | 21 | status_message = """Users by states: 22 | {states} 23 | 24 | Users by kyc verification result: 25 | {kycs} 26 | 27 | Users submitted their IDs: {uploads} 28 | 29 | Users by ID verification: 30 | {ids} 31 | """ 32 | 33 | onfido_message = """ 34 | Number of attempted users: {users} 35 | Number of created checks: {checks} 36 | Number of processed checks: {completed} ({completed_percent:.3}%) 37 | 38 | Users by results: 39 | {results} 40 | """ 41 | 42 | 43 | @restricted 44 | def status(_: telegram.Bot, update: telegram.Update): 45 | states = User.objects.aggregate({'$group': {'_id': '$state', 'count': {'$sum': 1}}}) 46 | kyc = User.objects.aggregate({'$group': {'_id': '$kyc_result', 'count': {'$sum': 1}}}) 47 | idm = User.objects.aggregate({'$group': {'_id': '$idm_result', 'count': {'$sum': 1}}}) 48 | uploads = IDUpload.objects.count() 49 | 50 | update.message.reply_text( 51 | status_message.format( 52 | states='\n'.join(f'* {item["_id"]}: {item["count"]}' for item in states), 53 | kycs='\n'.join(f'* {item["_id"]}: {item["count"]}' for item in kyc), 54 | ids='\n'.join(f'* {item["_id"]}: {item["count"]}' for item in idm), 55 | uploads=uploads, 56 | ), 57 | parse_mode=telegram.ParseMode.HTML, 58 | quote=False, 59 | ) 60 | 61 | 62 | @restricted 63 | def onfido(_: telegram.Bot, update: telegram.Update): 64 | states = User.objects.aggregate({'$group': {'_id': '$onfid_status', 'count': {'$sum': 1}}}) 65 | count = User.objects(onfid_id__ne=None).count() 66 | checks = Check.objects.count() 67 | 68 | res = {item['_id']: item['count'] for item in states if item['_id']} 69 | completed = sum(res.values()) 70 | 71 | update.message.reply_text( 72 | onfido_message.format( 73 | results='\n'.join(f'* {key}: {value} ({value/completed*100.:.3}%)' for key, value in res.items()), 74 | completed=completed, 75 | completed_percent=float(completed / checks * 100.), 76 | checks=checks, 77 | users=count, 78 | ), 79 | parse_mode=telegram.ParseMode.HTML, 80 | quote=False, 81 | ) 82 | 83 | 84 | @restricted 85 | def countries(_: telegram.Bot, update: telegram.Update): 86 | agg = User.objects.aggregate({'$group': {'_id': '$country_code', 'count': {'$sum': 1}}}) 87 | values = [ 88 | (item['_id'], item['count']) 89 | for item in sorted(agg, key=lambda x: x['count'], reverse=True) 90 | ] 91 | 92 | update.message.reply_text( 93 | "\n".join([f'* {key}: {value}' for key, value in values[:15]]), 94 | parse_mode=telegram.ParseMode.HTML, 95 | quote=False, 96 | ) 97 | 98 | 99 | @restricted 100 | def count(_: telegram.Bot, update: telegram.Update): 101 | users = User.objects.count() 102 | update.message.reply_text( 103 | f'There are *{users}* registered users!', 104 | parse_mode=telegram.ParseMode.MARKDOWN, 105 | quote=False, 106 | ) 107 | 108 | 109 | @restricted 110 | def do_export(_: telegram.Bot, update: telegram.Update, args: list = None): 111 | state = args[0] if args else None 112 | 113 | if state and state not in ALL_STATES: 114 | update.message.reply_text( 115 | f'Invalid state. Possible values are: {", ".join(ALL_STATES)}' 116 | ) 117 | return 118 | 119 | if update.effective_chat.id != update.effective_user.id: 120 | update.message.reply_text('Please send this command directly to bot (in a private channel)') 121 | return 122 | 123 | gevent.spawn(export_async, update, state) 124 | 125 | 126 | def export_async(update: telegram.Update, state: str): 127 | now = datetime.datetime.utcnow().isoformat() 128 | filename = f'Export.{now}.{ObjectId()}.csv' 129 | signed_url = s3.sign_url(filename, method='GET', expire=1800) 130 | 131 | with StringIO() as fh: 132 | export_count = export(fh, state=state) 133 | fh.seek(0) 134 | 135 | Upload.upload(fh.read(), filename, 'text/csv') 136 | update.message.reply_text( 137 | f'Exported *{export_count}* users to {signed_url}', 138 | parse_mode=telegram.ParseMode.MARKDOWN, 139 | quote=False, 140 | ) 141 | 142 | 143 | @restricted 144 | def info(_: telegram.Bot, update: telegram.Update, args: list = None): 145 | if not args: 146 | update.message.reply_text("Please provide user id") 147 | return 148 | 149 | try: 150 | user = User.objects(id=ObjectId(args[0])).first() 151 | except: 152 | try: 153 | user = User.objects(eth_address=to_eth(args[0])).first() 154 | except: 155 | user = User.objects(Q(email=args[0]) | Q(telegram=args[0])).first() 156 | 157 | if not user: 158 | update.message.reply_text( 159 | "I wasn't able to find such user", 160 | parse_mode=telegram.ParseMode.HTML, 161 | quote=True, 162 | ) 163 | return 164 | 165 | update.message.reply_text( 166 | "\n".join([ 167 | f'{key}: {value}' 168 | for key, value in user.to_csv().items() 169 | ]), 170 | parse_mode=telegram.ParseMode.HTML, 171 | quote=True, 172 | ) 173 | 174 | 175 | handlers = [ 176 | CommandHandler('count', count), 177 | CommandHandler('countries', countries), 178 | CommandHandler('status', status), 179 | CommandHandler('onfido', onfido), 180 | CommandHandler('export', do_export, pass_args=True), 181 | CommandHandler('info', info, pass_args=True), 182 | ] 183 | 184 | for handler in handlers: 185 | dispatcher.add_handler(handler) 186 | -------------------------------------------------------------------------------- /app/bot/helpers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from functools import wraps 3 | 4 | import requests 5 | import telegram 6 | 7 | from config import TELEGRAM_ADMINS, TELEGRAM_TOKEN 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | def register_webhook(baseurl: str, append_token=True) -> requests.Response: 13 | return requests.post( 14 | f'https://api.telegram.org/bot{TELEGRAM_TOKEN}/setWebhook', 15 | json={ 16 | 'url': f'{baseurl}{TELEGRAM_TOKEN if append_token else ""}', 17 | } 18 | ) 19 | 20 | 21 | def get_webhook_info() -> dict: 22 | return requests.get(f'https://api.telegram.org/bot{TELEGRAM_TOKEN}/getWebhookInfo').json() 23 | 24 | 25 | def restricted(func): 26 | @wraps(func) 27 | def wrapped(bot, update, *args, **kwargs): 28 | user_id = update.effective_user.id 29 | chat_id = update.effective_chat.id 30 | if user_id not in TELEGRAM_ADMINS: 31 | log.error("Unauthorized access denied for %s in channel %s.", user_id, chat_id) 32 | return 33 | return catch_error(func)(bot, update, *args, **kwargs) 34 | 35 | return wrapped 36 | 37 | 38 | def catch_error(func): 39 | @wraps(func) 40 | def wrapper(bot, update, *args, **kwargs): 41 | try: 42 | func(bot, update, *args, **kwargs) 43 | except Exception as ex: 44 | log.exception(ex) 45 | update.message.reply_text( 46 | f"Snap! bad thing just happened:\n> {ex}", 47 | parse_mode=telegram.ParseMode.MARKDOWN, 48 | ) 49 | 50 | return wrapper 51 | -------------------------------------------------------------------------------- /app/bot/signals.py: -------------------------------------------------------------------------------- 1 | import telegram 2 | 3 | from bot.client import bot 4 | from config import TELEGRAM_ADMIN_CHANNEL, TELEGRAM_PUBLIC_CHANNEL 5 | from idm.models import IDMResponse 6 | from signals import Transition, connect, transition, log_exception 7 | from user.models import User 8 | from user.state import ID_DECLINED, ID_FAILED, ID_VERIFIED, INFO_FAILED, INFO_NOT_VERIFIED 9 | 10 | 11 | # @transition(None, ID_VERIFIED) 12 | @log_exception 13 | def public_stats_announcements(user: User, transition: Transition): 14 | if not TELEGRAM_PUBLIC_CHANNEL: 15 | return 16 | 17 | count = User.objects(state=ID_VERIFIED).count() 18 | if count % 1000: 19 | bot.send_message( 20 | TELEGRAM_PUBLIC_CHANNEL, 21 | f"We've reached *{count}* verified users!", 22 | parse_mode=telegram.ParseMode.MARKDOWN, 23 | ) 24 | 25 | 26 | # @transition(None, INFO_NOT_VERIFIED) 27 | @log_exception 28 | def new_user_registered(user: User, transition: Transition): 29 | if not TELEGRAM_ADMIN_CHANNEL: 30 | return 31 | 32 | bot.send_message( 33 | TELEGRAM_ADMIN_CHANNEL, 34 | f"User {user.name} {user.email}[{user.id}] just registered", 35 | parse_mode=telegram.ParseMode.HTML, 36 | ) 37 | 38 | 39 | # @transition(None, ID_DECLINED) 40 | @log_exception 41 | def declined_id_verification(user: User, transition: Transition): 42 | if not TELEGRAM_ADMIN_CHANNEL: 43 | return 44 | 45 | bot.send_message( 46 | TELEGRAM_ADMIN_CHANNEL, 47 | f"User {user} declined ID verification", 48 | parse_mode=telegram.ParseMode.MARKDOWN, 49 | ) 50 | 51 | 52 | # @transition(None, ID_FAILED) 53 | @log_exception 54 | def failed_id_verification(user: User, transition: Transition): 55 | if not TELEGRAM_ADMIN_CHANNEL: 56 | return 57 | 58 | bot.send_message( 59 | TELEGRAM_ADMIN_CHANNEL, 60 | f"Error during ID verification for user {user}", 61 | parse_mode=telegram.ParseMode.MARKDOWN, 62 | ) 63 | 64 | 65 | # @transition(None, INFO_FAILED) 66 | @log_exception 67 | def failed_id_verification(user: User, transition: Transition): 68 | if not TELEGRAM_ADMIN_CHANNEL: 69 | return 70 | 71 | bot.send_message( 72 | TELEGRAM_ADMIN_CHANNEL, 73 | f"Error during info verification for user {user}", 74 | parse_mode=telegram.ParseMode.MARKDOWN, 75 | ) 76 | 77 | 78 | @connect(IDMResponse.on_received) 79 | def notify_about_missing_user(response: IDMResponse, **_): 80 | if not response.user: 81 | bot.send_message( 82 | TELEGRAM_ADMIN_CHANNEL, 83 | f"Response {response.id} ({response.transaction_id}) have no user associated", 84 | parse_mode=telegram.ParseMode.MARKDOWN, 85 | ) 86 | -------------------------------------------------------------------------------- /app/bot/views.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from flask import Blueprint, jsonify, request 4 | from telegram import Update 5 | 6 | from bot.client import bot, dispatcher 7 | from config import TELEGRAM_TOKEN 8 | 9 | log = logging.getLogger(__name__) 10 | blueprint = Blueprint('bot', __name__) 11 | 12 | 13 | @blueprint.route(f'/v1/bot/{TELEGRAM_TOKEN}', methods=['POST']) 14 | def handle_update(): 15 | update = Update.de_json(request.json, bot) 16 | dispatcher.process_update(update) 17 | return jsonify({'status': 'ok'}) 18 | -------------------------------------------------------------------------------- /app/chunks.py: -------------------------------------------------------------------------------- 1 | def chunks(iterable, chunk_size): 2 | """ 3 | breaks iterable on evenly sized chunks 4 | """ 5 | it = iter(iterable) 6 | while True: 7 | chunk = [] 8 | for i in range(chunk_size): 9 | try: 10 | chunk.append(next(it)) 11 | except StopIteration: 12 | break 13 | if chunk: 14 | yield chunk 15 | else: 16 | break 17 | -------------------------------------------------------------------------------- /app/config/__init__.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | 4 | DEBUG = False 5 | 6 | CURRENT_RELEASE = os.environ.get('CURRENT_RELEASE') 7 | CURRENT_ENVIRONMENT = os.environ.get('CURRENT_ENVIRONMENT') 8 | 9 | LOGGING = { 10 | 'version': 1, 11 | 'disable_existing_loggers': False, # this fixes the problem 12 | 'formatters': { 13 | 'standard': { 14 | 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' 15 | }, 16 | }, 17 | 'handlers': { 18 | 'default': { 19 | 'level': 'NOTSET', 20 | 'class': 'logging.StreamHandler', 21 | }, 22 | }, 23 | 'loggers': { 24 | '': { 25 | 'handlers': ['default'], 26 | 'level': 'NOTSET', 27 | 'propagate': True 28 | } 29 | } 30 | } 31 | 32 | DOCK_PER_ETH = 1 / 0.00009333 # Amount of DOCK tokens one can get for a single ETH 33 | 34 | 35 | with open('eths.txt', 'r') as fh: 36 | WHITELISTED_ADDRESSES = {itm.strip() for itm in fh} 37 | 38 | ETH_BALANCE_ADDRESS = os.environ.get('ETH_BALANCE_ADDRESS', '') 39 | ETH_ADDRESS = os.environ.get('ETH_ADDRESS') 40 | ETH_MAX_CONTRIBUTION = float(os.environ.get('ETH_MAX_CONTRIBUTION', '0.01')) 41 | 42 | ONFIDO_TOKEN = os.environ.get('ONFIDO_TOKEN') 43 | IDM_USERNAME = os.environ.get('IDM_USERNAME') 44 | IDM_PASSWORD = os.environ.get('IDM_PASSWORD') 45 | IDM_URL = 'https://edna.identitymind.com/im/account/consumer' # PROD 46 | # IDM_URL = 'https://staging.identitymind.com/im/account/consumer' # STAGING 47 | # IDM_URL = 'https://sandbox.identitymind.com/im/account/consumer' # SANDBOX 48 | IDM_WEBHOOK_USERNAME = os.environ.get('IDM_WEBHOOK_USERNAME') 49 | IDM_WEBHOOK_PASSWORD = os.environ.get('IDM_WEBHOOK_PASSWORD') 50 | 51 | VERIFIED_IDS_CAP = os.environ.get('VERIFIED_IDS_CAP', 25000) # Don't submit ids to idm after we've reached this cap 52 | 53 | AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') 54 | AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') 55 | AWS_BUCKET = os.environ.get('AWS_BUCKET') 56 | 57 | CUSTOMER_IO_SITE_ID = os.environ.get('CUSTOMER_IO_SITE_ID') 58 | CUSTOMER_IO_API_KEY = os.environ.get('CUSTOMER_IO_API_KEY') 59 | 60 | TELEGRAM_TOKEN = os.environ.get('TELEGRAM_TOKEN') 61 | TELEGRAM_ADMINS = [int(item) for item in os.environ.get('TELEGRAM_ADMINS', '').split(',') if item] 62 | TELEGRAM_ADMIN_CHANNEL = os.environ.get('TELEGRAM_ADMIN_CHANNEL') 63 | TELEGRAM_PUBLIC_CHANNEL = os.environ.get('TELEGRAM_PUBLIC_CHANNEL') 64 | 65 | WHITELIST_CLOSED = os.environ.get('WHITELIST_CLOSED', False) in ['yes', 'true', '1', 'y', 't'] 66 | 67 | # UTC timestamp of when whitelist should be opened 68 | WHITELIST_OPEN_DATE = os.environ.get('WHITELIST_OPEN_TS', None) 69 | if WHITELIST_OPEN_DATE: 70 | try: 71 | WHITELIST_OPEN_DATE = datetime.datetime.utcfromtimestamp(int(WHITELIST_OPEN_DATE)) 72 | except: 73 | WHITELIST_OPEN_DATE = None 74 | 75 | SALT = os.environ.get('SALT', 'salt') 76 | -------------------------------------------------------------------------------- /app/config/prod.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | MONGODB_URI = os.environ.get('MONGODB_URI') 4 | SENTRY_DSN = os.environ.get('SENTRY_DSN') 5 | 6 | TESTING = False 7 | DEBUG = False 8 | 9 | SALT = os.environ.get('SALT') 10 | -------------------------------------------------------------------------------- /app/config/testing.py: -------------------------------------------------------------------------------- 1 | MONGODB_URI = 'mongomock://localhost' 2 | 3 | TESTING = True 4 | DEBUG = True 5 | 6 | SALT = 'salt' 7 | -------------------------------------------------------------------------------- /app/conftest.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from collections import OrderedDict 3 | from typing import Callable, Dict, List, Type 4 | from urllib.parse import urlencode 5 | 6 | import flask.testing 7 | import pytest 8 | from bson import ObjectId 9 | from flask import Flask, Response, json 10 | from mongoengine import Document 11 | 12 | from app import create_app 13 | from errors import AppError 14 | from tokens import get_token 15 | from user.models import User 16 | 17 | 18 | @pytest.fixture(scope='function') 19 | def app() -> Flask: 20 | yield create_app('testing') 21 | 22 | # DB cleanups after each run 23 | db = Document._get_db() 24 | for name in db._collections.keys(): 25 | db._collections[name]._documents = OrderedDict() 26 | 27 | 28 | class TestClient(object): 29 | def __init__(self, client: flask.testing.FlaskClient): 30 | self.client = client 31 | 32 | def send(self, method: str, url: str, query: Dict = None, data: Dict = None, auth: str = None, 33 | **headers) -> Response: 34 | if auth: 35 | headers.update({'Authorization': f'Basic {auth}'}) 36 | return self.client.open( 37 | method=method, 38 | path=url, 39 | query_string=urlencode(query) if query else None, 40 | content_type='application/json', 41 | data=json.dumps(data), 42 | headers=headers, 43 | ) 44 | 45 | def delete(self, url: str, query: Dict = None, data: Dict = None, auth: str = None, **headers) -> flask.Response: 46 | return self.send('DELETE', url, query=query, data=data, auth=auth, **headers) 47 | 48 | def get(self, url: str, query: Dict = None, data: Dict = None, auth: str = None, **headers) -> flask.Response: 49 | return self.send('GET', url, query=query, data=data, auth=auth, **headers) 50 | 51 | def post(self, url: str, data: Dict = None, query: Dict = None, auth: str = None, **headers) -> flask.Response: 52 | return self.send('POST', url, query=query, data=data, auth=auth, **headers) 53 | 54 | def put(self, url: str, data: Dict = None, query: Dict = None, auth: str = None, **headers) -> flask.Response: 55 | return self.send('PUT', url, query=query, data=data, auth=auth, **headers) 56 | 57 | 58 | @pytest.fixture 59 | def service(client: flask.testing.FlaskClient) -> TestClient: 60 | return TestClient(client) 61 | 62 | 63 | @pytest.fixture 64 | def read(service: TestClient) -> TestClient: 65 | return service 66 | 67 | 68 | def error(response: flask.Response, error: Type[AppError]) -> bool: 69 | """ Assert that given response is an instance of given error object """ 70 | assert response.status_code == error.code, response.status_code 71 | assert response.json.get('error') == error.slugify_exception_name(), response.json 72 | return True 73 | 74 | 75 | @pytest.fixture() 76 | def user(service) -> User: 77 | return User( 78 | email='blap@example.com', 79 | eth_address='0000000000000000000000000000000000000000', 80 | telegram='telegram', 81 | dob=datetime.datetime.utcnow(), 82 | ).save() 83 | 84 | 85 | @pytest.fixture 86 | def three_users(service) -> List[User]: 87 | return [user(service) for _ in range(3)] 88 | 89 | 90 | @pytest.fixture 91 | def token() -> Callable: 92 | def inner(user: User = None) -> str: 93 | return get_token(user.id if user else ObjectId()) 94 | 95 | return inner 96 | -------------------------------------------------------------------------------- /app/customerio/__init__.py: -------------------------------------------------------------------------------- 1 | import customerio.signals 2 | -------------------------------------------------------------------------------- /app/customerio/client.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module provides communication with Customer.io api. 3 | 4 | http://customer.io/docs/api/rest.html 5 | 6 | Using some code from official Python implementation: 7 | https://github.com/customerio/customerio-python 8 | """ 9 | 10 | import json 11 | import logging 12 | from typing import Optional, Union 13 | 14 | from bson import ObjectId 15 | from requests import HTTPError, Response 16 | 17 | from config import CUSTOMER_IO_API_KEY, CUSTOMER_IO_SITE_ID 18 | from session import build_session 19 | from tokens import get_token 20 | from user.models import User 21 | 22 | UserOrId = Union[User, ObjectId, str] 23 | 24 | log = logging.getLogger('customerio') 25 | 26 | 27 | class CustomerIO(object): 28 | """ Customer.io api """ 29 | 30 | URL_PREFIX = 'https://track.customer.io/api/v1/customers' 31 | json_encoder = json.JSONEncoder 32 | _session = None 33 | 34 | @property 35 | def session(self): 36 | if not self._session and not CUSTOMER_IO_API_KEY: 37 | return 38 | 39 | if not self._session: 40 | ses = build_session(1000) 41 | ses.auth = (CUSTOMER_IO_SITE_ID, CUSTOMER_IO_API_KEY) 42 | ses.headers.update({'Content-Type': 'application/json'}) 43 | self._session = ses 44 | return self._session 45 | 46 | def request(self, method: str, user_or_id: UserOrId, section: str = None, **body) -> Optional[Response]: 47 | """ Make request to Customer.io API. """ 48 | if not self.session: 49 | return 50 | 51 | customer_id = str(getattr(user_or_id, 'id', user_or_id)) 52 | url_bits = [self.URL_PREFIX, customer_id] 53 | if section: 54 | url_bits.append(section) 55 | url = '/'.join(url_bits) 56 | try: 57 | req = self.session.request(method, url, json=body) 58 | req.raise_for_status() 59 | return req 60 | except HTTPError as err: 61 | log.exception('%s - %s', err, err.response.content) 62 | return err.response 63 | 64 | # === User === 65 | 66 | def identify(self, user: UserOrId, **kwargs): 67 | """ Identify a single customer by their unique id, and optionally add attributes. """ 68 | data = user.to_csv() 69 | data.update( 70 | email=user.email, 71 | telegram=user.telegram, 72 | created_at=int(user.created_at.strftime('%s')), 73 | dob=int(user.dob.strftime('%s')) if user.dob else None, 74 | token=get_token(user.id), 75 | ) 76 | data.update(kwargs) 77 | return self.request('PUT', user, **data) 78 | 79 | def delete(self, user: UserOrId): 80 | """ Delete a customer profile. """ 81 | return self.request('DELETE', user) 82 | 83 | # === Event == 84 | 85 | def event(self, user: UserOrId, event_name: str, **data): 86 | """ Track an event for a given customer_id """ 87 | return self.request('POST', user, 'events', name=event_name, data=data) 88 | 89 | 90 | customerio = CustomerIO() 91 | -------------------------------------------------------------------------------- /app/customerio/conftest.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from typing import Callable 3 | 4 | import pytest 5 | import requests_mock 6 | from mock import Mock, patch 7 | 8 | from session import build_session 9 | from user.models import User 10 | 11 | 12 | @pytest.fixture 13 | def customer_request(service) -> Mock: 14 | with patch('customerio.client.CustomerIO._session') as sess: 15 | sess.request = Mock() 16 | yield sess.request 17 | 18 | 19 | @pytest.fixture 20 | def customer_identify(service) -> Callable: 21 | @contextmanager 22 | def inner(user: User, **data) -> requests_mock.Mocker: 23 | with patch('customerio.client.CustomerIO.session', build_session()): 24 | with requests_mock.mock() as mock: 25 | mock.put( 26 | f'https://track.customer.io/api/v1/customers/{user.id}', 27 | json=data or user.to_json(), 28 | ) 29 | yield mock 30 | 31 | return inner 32 | 33 | 34 | @pytest.fixture 35 | def customer_event(service) -> Callable: 36 | @contextmanager 37 | def inner(user: User, event_name: str = None, **data) -> requests_mock.Mocker: 38 | with patch('customerio.client.CustomerIO.session', build_session()): 39 | with requests_mock.mock() as mock: 40 | mock.post( 41 | f'https://track.customer.io/api/v1/customers/{user.id}/events', 42 | json={'name': event_name, 'data': data} if event_name else None 43 | ) 44 | yield mock 45 | 46 | return inner 47 | 48 | 49 | class CustomerMock(requests_mock.Mocker): 50 | def __init__(self, user: User): 51 | super().__init__() 52 | self._user = user 53 | 54 | @property 55 | def identified(self): 56 | for history in self.request_history: 57 | if f'https://track.customer.io/api/v1/customers/{self._user.id}' == history.url: 58 | return True 59 | return False 60 | 61 | def has_event(self, event_name: str, data: dict = None): 62 | for history in self.request_history: 63 | if f'https://track.customer.io/api/v1/customers/{self._user.id}/events' != history.url: 64 | continue 65 | body = history.json() 66 | if body.get('name') != event_name: 67 | continue 68 | 69 | if data and body.get('data'): 70 | if data == body.get('data'): 71 | return True 72 | return False 73 | 74 | 75 | @pytest.fixture 76 | def customer(service) -> Callable: 77 | @contextmanager 78 | def inner(user: User) -> CustomerMock: 79 | with patch('customerio.client.CustomerIO.session', build_session()): 80 | with CustomerMock(user) as mock: 81 | mock.post(f'https://track.customer.io/api/v1/customers/{user.id}/events') 82 | mock.put(f'https://track.customer.io/api/v1/customers/{user.id}') 83 | yield mock 84 | 85 | return inner 86 | -------------------------------------------------------------------------------- /app/customerio/const.py: -------------------------------------------------------------------------------- 1 | EVENT_TRANSITION = 'transition' 2 | EVENT_KYC_APPROVED = 'kyc-approved' 3 | -------------------------------------------------------------------------------- /app/customerio/geo.py: -------------------------------------------------------------------------------- 1 | name_by_iso = { 2 | 'AF': 'Afghanistan', 3 | 'AX': 'Aland Islands', 4 | 'AL': 'Albania', 5 | 'DZ': 'Algeria', 6 | 'AS': 'American Samoa', 7 | 'AD': 'Andorra', 8 | 'AO': 'Angola', 9 | 'AI': 'Anguilla', 10 | 'AQ': 'Antarctica', 11 | 'AG': 'Antigua And Barbuda', 12 | 'AR': 'Argentina', 13 | 'AM': 'Armenia', 14 | 'AW': 'Aruba', 15 | 'AU': 'Australia', 16 | 'AT': 'Austria', 17 | 'AZ': 'Azerbaijan', 18 | 'BS': 'Bahamas', 19 | 'BH': 'Bahrain', 20 | 'BD': 'Bangladesh', 21 | 'BB': 'Barbados', 22 | 'BY': 'Belarus', 23 | 'BE': 'Belgium', 24 | 'BZ': 'Belize', 25 | 'BJ': 'Benin', 26 | 'BM': 'Bermuda', 27 | 'BT': 'Bhutan', 28 | 'BO': 'Bolivia', 29 | 'BA': 'Bosnia And Herzegovina', 30 | 'BW': 'Botswana', 31 | 'BV': 'Bouvet Island', 32 | 'BR': 'Brazil', 33 | 'IO': 'British Indian Ocean Territory', 34 | 'BN': 'Brunei Darussalam', 35 | 'BG': 'Bulgaria', 36 | 'BF': 'Burkina Faso', 37 | 'BI': 'Burundi', 38 | 'KH': 'Cambodia', 39 | 'CM': 'Cameroon', 40 | 'CA': 'Canada', 41 | 'CV': 'Cape Verde', 42 | 'KY': 'Cayman Islands', 43 | 'CF': 'Central African Republic', 44 | 'TD': 'Chad', 45 | 'CL': 'Chile', 46 | 'CN': 'China', 47 | 'CX': 'Christmas Island', 48 | 'CC': 'Cocos (Keeling) Islands', 49 | 'CO': 'Colombia', 50 | 'KM': 'Comoros', 51 | 'CG': 'Congo', 52 | 'CD': 'Congo, Democratic Republic', 53 | 'CK': 'Cook Islands', 54 | 'CR': 'Costa Rica', 55 | 'CI': 'Cote D\'Ivoire', 56 | 'HR': 'Croatia', 57 | 'CU': 'Cuba', 58 | 'CY': 'Cyprus', 59 | 'CZ': 'Czech Republic', 60 | 'DK': 'Denmark', 61 | 'DJ': 'Djibouti', 62 | 'DM': 'Dominica', 63 | 'DO': 'Dominican Republic', 64 | 'EC': 'Ecuador', 65 | 'EG': 'Egypt', 66 | 'SV': 'El Salvador', 67 | 'GQ': 'Equatorial Guinea', 68 | 'ER': 'Eritrea', 69 | 'EE': 'Estonia', 70 | 'ET': 'Ethiopia', 71 | 'FK': 'Falkland Islands (Malvinas)', 72 | 'FO': 'Faroe Islands', 73 | 'FJ': 'Fiji', 74 | 'FI': 'Finland', 75 | 'FR': 'France', 76 | 'GF': 'French Guiana', 77 | 'PF': 'French Polynesia', 78 | 'TF': 'French Southern Territories', 79 | 'GA': 'Gabon', 80 | 'GM': 'Gambia', 81 | 'GE': 'Georgia', 82 | 'DE': 'Germany', 83 | 'GH': 'Ghana', 84 | 'GI': 'Gibraltar', 85 | 'GR': 'Greece', 86 | 'GL': 'Greenland', 87 | 'GD': 'Grenada', 88 | 'GP': 'Guadeloupe', 89 | 'GU': 'Guam', 90 | 'GT': 'Guatemala', 91 | 'GG': 'Guernsey', 92 | 'GN': 'Guinea', 93 | 'GW': 'Guinea-Bissau', 94 | 'GY': 'Guyana', 95 | 'HT': 'Haiti', 96 | 'HM': 'Heard Island & Mcdonald Islands', 97 | 'VA': 'Holy See (Vatican City State)', 98 | 'HN': 'Honduras', 99 | 'HK': 'Hong Kong', 100 | 'HU': 'Hungary', 101 | 'IS': 'Iceland', 102 | 'IN': 'India', 103 | 'ID': 'Indonesia', 104 | 'IR': 'Iran, Islamic Republic Of', 105 | 'IQ': 'Iraq', 106 | 'IE': 'Ireland', 107 | 'IM': 'Isle Of Man', 108 | 'IL': 'Israel', 109 | 'IT': 'Italy', 110 | 'JM': 'Jamaica', 111 | 'JP': 'Japan', 112 | 'JE': 'Jersey', 113 | 'JO': 'Jordan', 114 | 'KZ': 'Kazakhstan', 115 | 'KE': 'Kenya', 116 | 'KI': 'Kiribati', 117 | 'KR': 'Korea', 118 | 'KW': 'Kuwait', 119 | 'KG': 'Kyrgyzstan', 120 | 'LA': 'Lao People\'s Democratic Republic', 121 | 'LV': 'Latvia', 122 | 'LB': 'Lebanon', 123 | 'LS': 'Lesotho', 124 | 'LR': 'Liberia', 125 | 'LY': 'Libyan Arab Jamahiriya', 126 | 'LI': 'Liechtenstein', 127 | 'LT': 'Lithuania', 128 | 'LU': 'Luxembourg', 129 | 'MO': 'Macao', 130 | 'MK': 'Macedonia', 131 | 'MG': 'Madagascar', 132 | 'MW': 'Malawi', 133 | 'MY': 'Malaysia', 134 | 'MV': 'Maldives', 135 | 'ML': 'Mali', 136 | 'MT': 'Malta', 137 | 'MH': 'Marshall Islands', 138 | 'MQ': 'Martinique', 139 | 'MR': 'Mauritania', 140 | 'MU': 'Mauritius', 141 | 'YT': 'Mayotte', 142 | 'MX': 'Mexico', 143 | 'FM': 'Micronesia, Federated States Of', 144 | 'MD': 'Moldova', 145 | 'MC': 'Monaco', 146 | 'MN': 'Mongolia', 147 | 'ME': 'Montenegro', 148 | 'MS': 'Montserrat', 149 | 'MA': 'Morocco', 150 | 'MZ': 'Mozambique', 151 | 'MM': 'Myanmar', 152 | 'NA': 'Namibia', 153 | 'NR': 'Nauru', 154 | 'NP': 'Nepal', 155 | 'NL': 'Netherlands', 156 | 'AN': 'Netherlands Antilles', 157 | 'NC': 'New Caledonia', 158 | 'NZ': 'New Zealand', 159 | 'NI': 'Nicaragua', 160 | 'NE': 'Niger', 161 | 'NG': 'Nigeria', 162 | 'NU': 'Niue', 163 | 'NF': 'Norfolk Island', 164 | 'MP': 'Northern Mariana Islands', 165 | 'NO': 'Norway', 166 | 'OM': 'Oman', 167 | 'PK': 'Pakistan', 168 | 'PW': 'Palau', 169 | 'PS': 'Palestinian Territory, Occupied', 170 | 'PA': 'Panama', 171 | 'PG': 'Papua New Guinea', 172 | 'PY': 'Paraguay', 173 | 'PE': 'Peru', 174 | 'PH': 'Philippines', 175 | 'PN': 'Pitcairn', 176 | 'PL': 'Poland', 177 | 'PT': 'Portugal', 178 | 'PR': 'Puerto Rico', 179 | 'QA': 'Qatar', 180 | 'RE': 'Reunion', 181 | 'RO': 'Romania', 182 | 'RU': 'Russian Federation', 183 | 'RW': 'Rwanda', 184 | 'BL': 'Saint Barthelemy', 185 | 'SH': 'Saint Helena', 186 | 'KN': 'Saint Kitts And Nevis', 187 | 'LC': 'Saint Lucia', 188 | 'MF': 'Saint Martin', 189 | 'PM': 'Saint Pierre And Miquelon', 190 | 'VC': 'Saint Vincent And Grenadines', 191 | 'WS': 'Samoa', 192 | 'SM': 'San Marino', 193 | 'ST': 'Sao Tome And Principe', 194 | 'SA': 'Saudi Arabia', 195 | 'SN': 'Senegal', 196 | 'RS': 'Serbia', 197 | 'SC': 'Seychelles', 198 | 'SL': 'Sierra Leone', 199 | 'SG': 'Singapore', 200 | 'SK': 'Slovakia', 201 | 'SI': 'Slovenia', 202 | 'SB': 'Solomon Islands', 203 | 'SO': 'Somalia', 204 | 'ZA': 'South Africa', 205 | 'GS': 'South Georgia And Sandwich Isl.', 206 | 'ES': 'Spain', 207 | 'LK': 'Sri Lanka', 208 | 'SD': 'Sudan', 209 | 'SR': 'Suriname', 210 | 'SJ': 'Svalbard And Jan Mayen', 211 | 'SZ': 'Swaziland', 212 | 'SE': 'Sweden', 213 | 'CH': 'Switzerland', 214 | 'SY': 'Syrian Arab Republic', 215 | 'TW': 'Taiwan', 216 | 'TJ': 'Tajikistan', 217 | 'TZ': 'Tanzania', 218 | 'TH': 'Thailand', 219 | 'TL': 'Timor-Leste', 220 | 'TG': 'Togo', 221 | 'TK': 'Tokelau', 222 | 'TO': 'Tonga', 223 | 'TT': 'Trinidad And Tobago', 224 | 'TN': 'Tunisia', 225 | 'TR': 'Turkey', 226 | 'TM': 'Turkmenistan', 227 | 'TC': 'Turks And Caicos Islands', 228 | 'TV': 'Tuvalu', 229 | 'UG': 'Uganda', 230 | 'UA': 'Ukraine', 231 | 'AE': 'United Arab Emirates', 232 | 'GB': 'United Kingdom', 233 | 'US': 'United States', 234 | 'UM': 'United States Outlying Islands', 235 | 'UY': 'Uruguay', 236 | 'UZ': 'Uzbekistan', 237 | 'VU': 'Vanuatu', 238 | 'VE': 'Venezuela', 239 | 'VN': 'Viet Nam', 240 | 'VG': 'Virgin Islands, British', 241 | 'VI': 'Virgin Islands, U.S.', 242 | 'WF': 'Wallis And Futuna', 243 | 'EH': 'Western Sahara', 244 | 'YE': 'Yemen', 245 | 'ZM': 'Zambia', 246 | 'ZW': 'Zimbabwe', 247 | } -------------------------------------------------------------------------------- /app/customerio/signals.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from customerio.client import customerio 4 | from customerio.const import EVENT_TRANSITION 5 | from customerio.geo import name_by_iso 6 | from ids.const import IDS_NAMES 7 | from ids.models import IDUpload 8 | from onfid.const import CHECK_COMPLETE 9 | from onfid.models import Check 10 | from signals import Transition, connect 11 | from user.models import User 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | @connect(User.on_create) 17 | def identify_user(user: User, **_): 18 | try: 19 | customerio.identify(user, ip=user.ip, idm_tid=user.idm_tid) 20 | except: 21 | log.exception('Customer error') 22 | 23 | 24 | @connect(User.on_transition) 25 | def on_transition(user: User, transition: Transition): 26 | try: 27 | customerio.event(user, EVENT_TRANSITION, state_before=transition[0], state_now=transition[1]) 28 | except: 29 | log.exception('Customer transition error') 30 | 31 | 32 | @connect(IDUpload.on_create) 33 | def id_uploaded(upload: IDUpload, **_): 34 | try: 35 | customerio.identify( 36 | upload.user, 37 | doc_type=upload.doc_type, 38 | doct_type_name=IDS_NAMES.get(upload.doc_type), 39 | doc_county=upload.doc_country, 40 | doc_country_name=name_by_iso.get(upload.doc_country), 41 | doc_state=upload.doc_state, 42 | ) 43 | except: 44 | log.exception('Customer error') 45 | 46 | 47 | @connect(User.on_transition) 48 | def update_users_state(user: User, transition: Transition): 49 | try: 50 | customerio.identify(user) 51 | except: 52 | log.exception('Customer error') 53 | 54 | 55 | @connect(Check.on_update) 56 | def update_user_status_on_complete_check(check: Check, **_): 57 | if check.status != CHECK_COMPLETE: 58 | return 59 | 60 | if not check.user: 61 | log.exception('Check missing user: %s', check.id) 62 | return 63 | 64 | customerio.identify(check.user) 65 | 66 | 67 | @connect(Check.on_update) 68 | def store_check_event(check: Check, **_): 69 | if not check.user: 70 | log.exception('Check missing user: %s', check.id) 71 | return 72 | 73 | user = check.user 74 | customerio.event(user, 'onfido_check_update', **check.raw) 75 | -------------------------------------------------------------------------------- /app/customerio/test_customerio.py: -------------------------------------------------------------------------------- 1 | from customerio.client import customerio 2 | 3 | 4 | def test_identify(customer_identify, user): 5 | data = user.to_json() 6 | with customer_identify(user, blop='blop', **data) as call: 7 | customerio.identify(user, blip='blop') 8 | assert call.called 9 | 10 | 11 | def test_event(customer_event, user): 12 | with customer_event(user, 'event', blip='blop') as call: 13 | customerio.event(user, 'event', blip='blop') 14 | assert call.called 15 | -------------------------------------------------------------------------------- /app/customerio/test_signals.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from mock import patch 3 | 4 | from customerio.const import EVENT_TRANSITION 5 | from user.models import User 6 | from user.state import INFO_NOT_VERIFIED, NEW_USER 7 | from user.test_user_info import new_user 8 | 9 | 10 | def test_identify_new_user(service, customer): 11 | mocked_id = ObjectId() 12 | with patch('user.models.ObjectId') as oid: 13 | oid.return_value = mocked_id 14 | user = User(id=mocked_id) 15 | with customer(user) as call: 16 | res = service.post('/v1/user', new_user()) 17 | assert res.status_code == 201, res.json 18 | assert call.identified 19 | assert call.has_event(EVENT_TRANSITION, {'state_before': NEW_USER, 'state_now': INFO_NOT_VERIFIED}) 20 | -------------------------------------------------------------------------------- /app/errors.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Any 3 | 4 | import mongoengine.errors 5 | import voluptuous 6 | from flask import jsonify 7 | from werkzeug.exceptions import HTTPException 8 | 9 | 10 | class AppError(HTTPException): 11 | code = 400 12 | 13 | def __init__(self, key: str = None, details: Any = None, code: int = None): 14 | super(AppError, self).__init__() 15 | self.key = key 16 | if code: 17 | self.code = code 18 | self.details = details 19 | 20 | @classmethod 21 | def slugify_exception_name(cls): 22 | return re.sub(r'(?<=[a-z])(?=[A-Z])', '-', cls.__name__).lower() 23 | 24 | def get_response(self, environ=None): 25 | return self.jsonify() 26 | 27 | def jsonify(self): 28 | error_obj = { 29 | 'error': self.slugify_exception_name(), 30 | 'key': self.key, 31 | 'details': self.details, 32 | } 33 | 34 | res = jsonify(error_obj) 35 | res.status_code = self.code 36 | 37 | return res 38 | 39 | 40 | class ObjectExists(AppError): 41 | pass 42 | 43 | 44 | class ValidationError(AppError): 45 | pass 46 | 47 | 48 | class RouteNotFound(AppError): 49 | pass 50 | 51 | 52 | class ObjectNotFound(AppError): 53 | code = 404 54 | pass 55 | 56 | 57 | class WhitelistClosed(AppError): 58 | code = 410 59 | 60 | 61 | def register_errors(app): 62 | @app.errorhandler(AppError) 63 | def handle_invalid_usage(error): 64 | return error.jsonify() 65 | 66 | @app.errorhandler(mongoengine.errors.NotUniqueError) 67 | def handle_duplicate_object(error): 68 | return ObjectExists().jsonify() 69 | 70 | @app.errorhandler(mongoengine.errors.DoesNotExist) 71 | def handle_missing_object(error): 72 | return ObjectNotFound().jsonify() 73 | 74 | @app.errorhandler(404) 75 | def route_not_found_error(error): 76 | return RouteNotFound().jsonify() 77 | 78 | @app.errorhandler(voluptuous.Invalid) 79 | def route_validation_error(error): 80 | return ValidationError(str(error)).jsonify() 81 | -------------------------------------------------------------------------------- /app/eth/batch.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import sys 4 | from typing import Tuple 5 | 6 | import requests 7 | from mongoengine import QuerySet 8 | 9 | from app import create_app 10 | from chunks import chunks 11 | from eth.models import Address 12 | from onfid.helpers import rate_limit 13 | from user.models import User 14 | 15 | _ = create_app('prod') 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | def get_users(start: int = None, limit: int = None) -> QuerySet: 20 | res = User.objects.order_by('id') 21 | if start: 22 | res = res.skip(start) 23 | if limit: 24 | res = res.limit(limit) 25 | 26 | return res.no_cache() 27 | 28 | 29 | @rate_limit(500) 30 | def check_eth(address: str) -> Tuple[float, int]: 31 | try: 32 | res = requests.get( 33 | 'https://api.infura.io/v1/jsonrpc/mainnet/eth_getBalance', 34 | params={'params': json.dumps([f'0x{address}', 'latest'])}, 35 | ) 36 | value = res.json().get('result') 37 | int_value = int(value, 16) 38 | balance = float(int_value / (10 ** 18)) 39 | except Exception as ex: 40 | log.exception('eth_getBalance: %s', ex) 41 | balance = 0. 42 | 43 | try: 44 | res = requests.get( 45 | 'https://api.infura.io/v1/jsonrpc/mainnet/eth_getTransactionCount', 46 | params={'params': json.dumps([f'0x{address}', 'latest'])}, 47 | ) 48 | value = res.json().get('result') 49 | txes = int(value, 16) 50 | except Exception as ex: 51 | log.exception('eth_getBalance: %s', ex) 52 | txes = 0 53 | 54 | return balance, txes 55 | 56 | 57 | def check_user(user: User) -> bool: 58 | eth = user.eth_address 59 | stored = Address.objects(address=eth).first() 60 | if not stored: 61 | balance, txes = check_eth(eth) 62 | stored = Address(address=eth, balance=balance, transactions=txes).save() 63 | log.info('User %s has balance of %s and %s transactions', user.id, balance, txes) 64 | else: 65 | log.info('User %s has balance of %s and %s transactions [CACHED]', user.id, stored.balance, stored.transactions) 66 | return stored.balance and stored.transactions 67 | 68 | 69 | def main(start: int, stop: int): 70 | users = get_users(start, stop) 71 | 72 | for chunk in chunks(users, 100): 73 | for user in chunk: 74 | check_user(user) 75 | 76 | 77 | if __name__ == '__main__': 78 | start = count = 0 79 | if len(sys.argv) == 3: 80 | start = int(sys.argv[1]) 81 | count = int(sys.argv[2]) 82 | elif len(sys.argv) == 2: 83 | start = int(sys.argv[1]) 84 | 85 | log.info('Doing items %s to %s', start, start + count) 86 | main(start, count) 87 | -------------------------------------------------------------------------------- /app/eth/models.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import Any, Union 3 | 4 | from mongoengine import DateTimeField, Document, DynamicField, StringField, FloatField, BooleanField, IntField 5 | 6 | 7 | class Cache(Document): 8 | meta = { 9 | 'indexes': [ 10 | {'fields': ['created_at'], 'expireAfterSeconds': 600}, 11 | ] 12 | } 13 | key = StringField(primary_key=True) 14 | created_at = DateTimeField(default=datetime.datetime.utcnow) 15 | value = DynamicField() 16 | 17 | @classmethod 18 | def find(cls, key: str) -> Union[None, 'Cache']: 19 | result = cls.objects(key=key).first() 20 | if result: 21 | return result 22 | return None 23 | 24 | @classmethod 25 | def set(cls, key: str, value: Any): 26 | cls.objects(key=key).update( 27 | value=value, 28 | created_at=datetime.datetime.utcnow(), 29 | upsert=True, 30 | ) 31 | 32 | 33 | class Address(Document): 34 | address = StringField(primary_key=True) 35 | 36 | balance = FloatField() 37 | transactions = IntField() 38 | valid = BooleanField() 39 | 40 | comment = StringField() -------------------------------------------------------------------------------- /app/eth/views.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import functools 3 | import json 4 | import logging 5 | from typing import Tuple 6 | 7 | import requests 8 | from flask import Blueprint, jsonify 9 | 10 | from config import DOCK_PER_ETH, ETH_ADDRESS, ETH_BALANCE_ADDRESS 11 | from customerio.client import customerio 12 | from eth.models import Cache 13 | from user.models import User 14 | from user.state import CONTRIBUTED 15 | from user.views import to_eth 16 | 17 | ETH_CACHE_KEY = 'eth_amount' 18 | ETH_LAST_TRANSACTION = 'eth_last_transaction' 19 | 20 | log = logging.getLogger(__name__) 21 | blueprint = Blueprint('eth', __name__) 22 | 23 | 24 | def get_amount() -> float: 25 | if not ETH_BALANCE_ADDRESS: 26 | return 0 27 | 28 | data = {'params': json.dumps([ETH_BALANCE_ADDRESS, 'latest'])} 29 | try: 30 | res = requests.get('https://api.infura.io/v1/jsonrpc/mainnet/eth_getBalance', params=data) 31 | value = res.json().get('result') 32 | int_value = int(value, 16) 33 | return float(int_value / (10 ** 18)) 34 | except: 35 | return 0. 36 | 37 | 38 | @functools.lru_cache(2) 39 | def get_cached_amount(_: int) -> Tuple[float, float, datetime.datetime]: 40 | last_contribution = 0 41 | cached_contribution = Cache.find(ETH_LAST_TRANSACTION) 42 | if cached_contribution: 43 | last_contribution = cached_contribution.value 44 | 45 | cached = Cache.find(ETH_CACHE_KEY) 46 | if not cached or cached.created_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=30): 47 | amount = get_amount() 48 | if amount: 49 | Cache.set(ETH_CACHE_KEY, amount) 50 | return amount, last_contribution, datetime.datetime.utcnow() 51 | else: 52 | return cached.value, last_contribution, cached.created_at 53 | 54 | 55 | @blueprint.route('/v1/eth', methods=['GET']) 56 | def eth_amount(): 57 | ts = int(datetime.datetime.utcnow().strftime('%s')) 58 | amount, last_amount, cache_date = get_cached_amount(int(ts / 30)) # Hack to cache data for 5 minutes 59 | return jsonify({ 60 | 'amount': amount, 61 | 'updated_at': int(cache_date.strftime('%s')), 62 | 'last_amount': last_amount, 63 | }) 64 | 65 | 66 | @blueprint.route('/v1/check_contributions', methods=['GET']) 67 | def check_contributions(): 68 | """ 69 | { 70 | "blockNumber": "1961866", 71 | "timeStamp": "1469624867", 72 | "hash": "0x545243f19ede50b8115e6165ffe509fde4bb1abc20f287cd8c49c97f39836efe", 73 | "nonce": "22", 74 | "blockHash": "0x9ba94fe0b81b32593fd547c39ccbbc2fc14b1bdde4ccc6dccb79e2a304280d50", 75 | "transactionIndex": "5", 76 | "from": "0xddbd2b932c763ba5b1b7ae3b362eac3e8d40121a", 77 | "to": "0x1bb0ac60363e320bc45fdb15aed226fb59c88e44", 78 | "value": "10600000000000000000000", 79 | "gas": "127964", 80 | "gasPrice": "20000000000", 81 | "isError": "0", 82 | "txreceipt_status": "", 83 | "input": "0x", 84 | "contractAddress": "", 85 | "cumulativeGasUsed": "227901", 86 | "gasUsed": "27964", 87 | "confirmations": "3140511" 88 | }, 89 | 90 | """ 91 | res = requests.get( 92 | f'http://api.etherscan.io/api?module=account&action=txlist&address={ETH_ADDRESS}' 93 | f'&startblock=5120812&endblock=99999999&sort=desc' 94 | ) 95 | data = res.json() 96 | result = data.get('result') 97 | total = updated = 0 98 | last_amount = 0 99 | for item in result: 100 | total += 1 101 | 102 | err = item.get('isError') 103 | if err != '0': 104 | continue 105 | 106 | to = item.get('to') 107 | if not to or to_eth(to) != to_eth(ETH_ADDRESS): 108 | continue 109 | sender = to_eth(item.get('from')) 110 | 111 | user = User.objects(eth_address=sender).first() 112 | if not user: 113 | continue 114 | 115 | tx = item.get('hash') 116 | if tx in user.contribution_tx: 117 | continue 118 | 119 | amount = int(item.get('value')) / (10. ** 18) 120 | if not amount: 121 | continue 122 | 123 | if total == 1: 124 | last_amount = user.contribution_amount 125 | 126 | dock_amount = amount * DOCK_PER_ETH 127 | 128 | user.contribution_amount += amount 129 | user.contribution_tx.append(tx) 130 | user.save() 131 | if user.state != CONTRIBUTED: 132 | user.transition(CONTRIBUTED) 133 | 134 | customerio.event( 135 | user, 136 | 'token_purchase', 137 | eth_amount=amount, 138 | dock_amount=round(dock_amount, 8), 139 | ) 140 | updated += 1 141 | 142 | if last_amount: 143 | Cache.set(ETH_LAST_TRANSACTION, last_amount) 144 | 145 | return jsonify({ 146 | 'status': 'ok', 147 | 'total': total, 148 | 'updated': updated, 149 | 'last_amount': last_amount, 150 | }) 151 | -------------------------------------------------------------------------------- /app/idm/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /app/idm/const.py: -------------------------------------------------------------------------------- 1 | STATUS_ACCEPTED = 'ACCEPT' 2 | STATUS_PENDING = 'MANUAL_REVIEW' 3 | STATUS_DECLINED = 'DENY' 4 | 5 | KYC_STATE_ACCEPTED = 'A' 6 | KYC_STATE_PENDING = 'R' 7 | KYC_STATE_DECLINED = 'D' 8 | 9 | STATUS_MAP = { 10 | KYC_STATE_ACCEPTED: STATUS_ACCEPTED, 11 | KYC_STATE_PENDING: STATUS_PENDING, 12 | KYC_STATE_DECLINED: STATUS_DECLINED, 13 | } 14 | 15 | USER_REPUTATION_TRUSTED = 'TRUSTED' 16 | USER_REPUTATION_UNKNOWN = 'UNKNOWN' 17 | USER_REPUTATION_SUSPICIOUS = 'SUSPICIOUS' 18 | USER_REPUTATION_BAD = 'BAD' 19 | -------------------------------------------------------------------------------- /app/idm/errors.py: -------------------------------------------------------------------------------- 1 | from requests import HTTPError 2 | 3 | 4 | class IDMError(Exception): 5 | def __init__(self, error: HTTPError): 6 | self.error = error 7 | super().__init__() 8 | 9 | @property 10 | def text(self): 11 | return self.error.response.text 12 | -------------------------------------------------------------------------------- /app/idm/helpers.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from bson import ObjectId 4 | 5 | from idm.const import STATUS_MAP 6 | from user.models import User 7 | 8 | 9 | def parse_response(response: dict): 10 | return { 11 | 'transaction_id': response['mtid'], 12 | 'result': response.get('res'), # STATUS_* 13 | 'kyc_state': STATUS_MAP[response['state']], # KYC_STATUS_* -> STATUS_* 14 | 'user_reputation': response.get('user'), # USER_REPUTATION_* 15 | 'fraud_result': response.get('frp'), # STATUS_* 16 | 'previous_reputation': response.get('upr'), # USER_REPUTATION_* 17 | } 18 | 19 | 20 | def build_request(user: User, **kwargs) -> dict: 21 | return { 22 | 'dob': user.dob.date().isoformat(), 23 | 'scanData': kwargs.get('image1'), 24 | 'backsideImageData': kwargs.get('image2'), 25 | 'stage': kwargs.get('stage', 1), 26 | 'bfn': user.first_name, 27 | 'bln': user.last_name, 28 | # 'title': user.name, 29 | 'tid': kwargs.get('transaction_id', str(ObjectId())), 30 | 'man': str(user.id), 31 | 'tea': user.email, 32 | 'ip': kwargs.get('ip', user.ip), 33 | 'dfp': user.dfp, 34 | 'dft': kwargs.get('dft', 'AU'), 35 | 'bsn': user.address, 36 | 'bco': user.country_code, 37 | 'bz': user.zip_code, 38 | 'bc': user.city, 39 | 'bs': user.state_code, 40 | 'tti': int(datetime.datetime.utcnow().strftime('%s')), 41 | 'accountCreationTime': int(user.created_at.strftime('%s')), 42 | 'phn': user.phone, 43 | 'memo1': user.eth_address, 44 | 'memo2': user.eth_amount, 45 | 'docCountry': kwargs.get('doc_country'), 46 | 'docState': kwargs.get('doc_state'), 47 | 'docType': kwargs.get('doc_type'), 48 | } 49 | -------------------------------------------------------------------------------- /app/idm/models.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | 4 | from blinker import Signal 5 | from bson import ObjectId 6 | from mongoengine import DateTimeField, DictField, Document, IntField, ReferenceField, StringField 7 | from requests import HTTPError 8 | 9 | from config import IDM_PASSWORD, IDM_URL, IDM_USERNAME 10 | from idm.const import STATUS_PENDING 11 | from idm.errors import IDMError 12 | from idm.helpers import build_request, parse_response 13 | from session import build_session 14 | from user.models import User 15 | 16 | log = logging.getLogger(__name__) 17 | session = build_session(1024) 18 | session.auth = (IDM_USERNAME, IDM_PASSWORD) 19 | 20 | 21 | class IDMRequest(Document): 22 | """ Keeps track of all KYC requests we've performed to the IDM. """ 23 | 24 | meta = { 25 | 'indexes': ['user', 'transaction_id'], 26 | } 27 | 28 | user = ReferenceField(User, required=True) 29 | created_at = DateTimeField(default=datetime.datetime.utcnow) 30 | requested_at = DateTimeField() 31 | 32 | #: Transaction id for the same user must be the same (when user goes from first to second stage) 33 | transaction_id = StringField(default=lambda: str(ObjectId()), unique=True) 34 | 35 | request_data = DictField() 36 | response_status = IntField() 37 | 38 | on_create = Signal() 39 | 40 | @classmethod 41 | def create(cls, user: User, **data) -> 'IDMRequest': 42 | body = build_request(user, **data) 43 | obj = cls( 44 | user=user, 45 | request_data=body, 46 | transaction_id=body['tid'], 47 | ).save() 48 | cls.on_create.send(obj) 49 | return obj 50 | 51 | def request(self) -> 'IDMResponse': 52 | if not IDM_USERNAME: 53 | return IDMResponse.empty() 54 | 55 | try: 56 | res = session.request('POST', IDM_URL, json=self.request_data) 57 | self.update( 58 | requested_at=datetime.datetime.utcnow(), 59 | response_status=res.status_code, 60 | ) 61 | 62 | if res.status_code != 200: 63 | raise HTTPError(res.text, response=res) 64 | return IDMResponse.from_response(res.json()) 65 | except HTTPError as err: 66 | log.exception('IDM request error: %s', err) 67 | raise IDMError(err) 68 | 69 | 70 | class IDMResponse(Document): 71 | """ Keeps track of all responses we've received from the IDM. """ 72 | 73 | meta = { 74 | 'indexes': ['user', 'transaction_id'], 75 | } 76 | 77 | user = ReferenceField(User) 78 | received_at = DateTimeField(default=datetime.datetime.utcnow) 79 | 80 | transaction_id = StringField(required=True) 81 | result = StringField() 82 | kyc_state = StringField() 83 | user_reputation = StringField() 84 | fraud_result = StringField() 85 | previous_reputation = StringField() 86 | 87 | response_data = DictField() 88 | 89 | on_received = Signal() 90 | 91 | @property 92 | def status(self): 93 | return self.result or self.kyc_state or self.fraud_result 94 | 95 | @classmethod 96 | def from_response(cls, json: dict) -> 'IDMResponse': 97 | """ Parse received response and stores it as a document. """ 98 | response = parse_response(json) 99 | transaction_id = response['transaction_id'] 100 | 101 | if transaction_id[:3] == 'csv': 102 | user = User.objects(id=transaction_id[3:]).first() 103 | else: 104 | related_request = IDMRequest.objects(transaction_id=transaction_id).first() 105 | user = related_request.user if related_request else None 106 | 107 | obj = cls( 108 | response_data=json, 109 | user=user if user else None, 110 | **response, 111 | ).save() 112 | cls.on_received.send(obj) 113 | return obj 114 | 115 | @property 116 | def request(self) -> IDMRequest: 117 | return IDMRequest.objects(transaction_id=self.transaction_id).first() 118 | 119 | @classmethod 120 | def empty(cls): 121 | return cls( 122 | user=None, 123 | transaction_id=str(ObjectId()), 124 | result=STATUS_PENDING, 125 | ) 126 | -------------------------------------------------------------------------------- /app/idm/verify.py: -------------------------------------------------------------------------------- 1 | from config import VERIFIED_IDS_CAP 2 | from idm.const import STATUS_ACCEPTED 3 | from idm.models import IDMRequest, IDMResponse 4 | from user.models import User 5 | from user.state import BANNED_COUNTRIES, ID_VERIFIED 6 | 7 | 8 | def verify(user: User, kyc: bool = True, **kwargs) -> IDMResponse: 9 | req = IDMRequest.create( 10 | user=user, 11 | stage=1 if kyc else 2, 12 | image1=kwargs.get('image1'), 13 | image2=kwargs.get('image2'), 14 | doc_country=kwargs.get('doc_country'), 15 | doc_state=kwargs.get('doc_state'), 16 | doc_type=kwargs.get('doc_type'), 17 | ) 18 | 19 | should_request = user.country_code and user.country_code not in BANNED_COUNTRIES 20 | 21 | if not kyc: # Only pass ids to idm if user successfully passed KYC 22 | should_request = should_request and user.kyc_result == STATUS_ACCEPTED 23 | 24 | # Hard cap on number of verified ids 25 | should_request = should_request and User.objects(state=ID_VERIFIED).count() < VERIFIED_IDS_CAP 26 | 27 | if should_request: 28 | return req.request() 29 | 30 | return IDMResponse.empty() 31 | -------------------------------------------------------------------------------- /app/idm/views.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import functools 3 | import logging 4 | from typing import Callable 5 | 6 | from flask import Blueprint, jsonify, request 7 | from werkzeug.exceptions import Unauthorized 8 | 9 | from config import IDM_WEBHOOK_PASSWORD, IDM_WEBHOOK_USERNAME 10 | from customerio.client import customerio 11 | from idm.models import IDMResponse 12 | from user.state import INFO_DECLINED, INFO_NOT_VERIFIED, INFO_PENDING_VERIFICATION, INFO_VERIFIED 13 | from user.verifications import apply_response 14 | 15 | log = logging.getLogger(__name__) 16 | blueprint = Blueprint('idm', __name__) 17 | 18 | 19 | def idm_auth(func: Callable) -> Callable: 20 | @functools.wraps(func) 21 | def inner(*args, **kwargs): 22 | auth = request.headers.get('Authorization') 23 | if not auth: 24 | raise Unauthorized() 25 | 26 | token = auth[6:] 27 | if not token: 28 | raise Unauthorized() 29 | 30 | if token != base64.b64encode(bytes(f'{IDM_WEBHOOK_USERNAME}:{IDM_WEBHOOK_PASSWORD}', 'ascii')).decode('ascii'): 31 | raise Unauthorized() 32 | 33 | return func(*args, **kwargs) 34 | 35 | return inner 36 | 37 | 38 | @blueprint.route('/v1/idm', methods=['POST']) 39 | @idm_auth 40 | def idm_webhook(): 41 | json = request.get_json(force=True) 42 | log.info(json) 43 | response = IDMResponse.from_response(json) 44 | 45 | if response.user: 46 | info_states = [INFO_NOT_VERIFIED, INFO_PENDING_VERIFICATION, INFO_VERIFIED, INFO_DECLINED] 47 | customerio.event(response.user, 'idm-webhook') 48 | apply_response(response.user, response, response.user.state in info_states) 49 | 50 | return jsonify({'status': 'ok'}) 51 | -------------------------------------------------------------------------------- /app/ids/const.py: -------------------------------------------------------------------------------- 1 | DRIVING_LICENCE = 'DL' 2 | PASSPORT = 'PP' 3 | ID_CARD = 'ID' 4 | RESIDENCE_PERMIT = 'RP' 5 | UTILITY_BILL = 'UB' 6 | 7 | IDS_NAMES = { 8 | DRIVING_LICENCE: 'Driving licence', 9 | PASSPORT: 'Passport', 10 | ID_CARD: 'ID card', 11 | RESIDENCE_PERMIT: 'Residence permit', 12 | UTILITY_BILL: 'Utility bill', 13 | } 14 | 15 | DOC_TYPES = [ 16 | DRIVING_LICENCE, 17 | PASSPORT, 18 | ID_CARD, 19 | RESIDENCE_PERMIT, 20 | # UTILITY_BILL 21 | ] 22 | -------------------------------------------------------------------------------- /app/ids/models.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | from blinker import Signal 4 | from mongoengine import DateTimeField, Document, ReferenceField, StringField 5 | 6 | from ids.const import DOC_TYPES 7 | from upload.models import Upload 8 | from user.models import User 9 | 10 | 11 | class IDUpload(Document): 12 | user = ReferenceField(User, required=True) 13 | upload1 = ReferenceField(Upload, required=True) 14 | upload2 = ReferenceField(Upload, required=True) 15 | 16 | # TODO: Validate doc_country 17 | doc_type = StringField(required=True, choices=DOC_TYPES) 18 | doc_country = StringField(required=True) 19 | doc_state = StringField() 20 | 21 | created_at = DateTimeField() 22 | submitted_at = DateTimeField() 23 | verified_at = DateTimeField() 24 | status = StringField() #: Verification status 25 | 26 | on_create = Signal() 27 | on_verification_started = Signal() 28 | on_verification_completed = Signal() 29 | 30 | @classmethod 31 | def create(cls, user: User, upload1: Upload, upload2: Upload, doc_type: str, doc_country: str, 32 | doc_state: str) -> 'IDUpload': 33 | obj = cls( 34 | doc_country=doc_country, 35 | doc_state=doc_state, 36 | doc_type=doc_type, 37 | upload1=upload1, 38 | upload2=upload2, 39 | user=user, 40 | ).save(force_insert=True) 41 | cls.on_create.send(obj) 42 | return obj 43 | 44 | def verify(self) -> Tuple[bool, str]: 45 | # idm.verify_ids() 46 | return True, 'blop' 47 | 48 | def to_json(self): 49 | return { 50 | 'id': str(self.id), 51 | 'upload1': str(self.upload1.id) if self.upload1 else None, 52 | 'upload2': str(self.upload2.id) if self.upload2 else None, 53 | 'doc_type': self.doc_type, 54 | 'doc_country': self.doc_country, 55 | 'doc_state': self.doc_state, 56 | 'created_at': int(self.created_at.strftime('%s')) if self.created_at else None, 57 | 'submitted_at': int(self.submitted_at.strftime('%s')) if self.submitted_at else None, 58 | 'verified_at': int(self.verified_at.strftime('%s')) if self.verified_at else None, 59 | 'status': self.status, 60 | } 61 | -------------------------------------------------------------------------------- /app/ids/test_ids.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | import pytest 4 | from bson import ObjectId 5 | from mock import patch 6 | 7 | from conftest import error 8 | from errors import ObjectNotFound, ValidationError 9 | from ids.const import PASSPORT 10 | from ids.models import IDUpload 11 | from upload.models import Upload 12 | from user.errors import InvalidState 13 | from user.models import User 14 | from user.state import ID_NOT_VERIFIED, INFO_VERIFIED 15 | 16 | 17 | @pytest.fixture 18 | def upload() -> Callable: 19 | def inner(user: User, **kwargs) -> Upload: 20 | return Upload.create( 21 | user=user, 22 | original_filename=kwargs.get('filename', 'example.jpg'), 23 | content_type=kwargs.get('content_type', 'image/jpg'), 24 | size=kwargs.get('size', 5 * 1024), 25 | ) 26 | 27 | return inner 28 | 29 | 30 | @pytest.fixture 31 | def ids_upload(upload) -> Callable: 32 | def inner(user: User, **kwargs) -> IDUpload: 33 | return IDUpload.create( 34 | user=user, 35 | upload1=upload(user), 36 | upload2=upload(user), 37 | doc_type=PASSPORT, 38 | doc_country='AU', 39 | doc_state='', 40 | ) 41 | 42 | return inner 43 | 44 | 45 | def test_create_package(service, user, token, upload): 46 | upload1 = upload(user) 47 | upload2 = upload(user) 48 | 49 | user.transition(INFO_VERIFIED) 50 | 51 | data = { 52 | 'upload1': str(upload1.id), 53 | 'upload2': str(upload2.id), 54 | 'doc_type': PASSPORT, 55 | 'doc_country': 'AU', 56 | 'doc_state': None, 57 | } 58 | with patch('upload.models.Upload.stored_size', 500 * 1024): 59 | res = service.post('/v1/ids', data, auth=token(user)) 60 | assert res.status_code == 201, res.json 61 | 62 | 63 | def test_invalid_doc_id(service, user, token, upload): 64 | upload1 = upload(user) 65 | upload2 = upload(user) 66 | 67 | user.transition(INFO_VERIFIED) 68 | 69 | data = { 70 | 'upload1': str(upload1.id), 71 | 'upload2': str(upload2.id), 72 | 'doc_type': 'TurtlePass', 73 | 'doc_country': 'AU', 74 | 'doc_state': '', 75 | } 76 | res = service.post('/v1/ids', data, auth=token(user)) 77 | assert res.status_code == 400 78 | 79 | 80 | def test_create_package_missing_upload(service, user, token, upload): 81 | upload1 = upload(user) 82 | 83 | user.transition(INFO_VERIFIED) 84 | 85 | data = { 86 | 'upload1': str(upload1.id), 87 | 'upload2': str(ObjectId()), 88 | 'doc_type': 'PP', 89 | 'doc_country': 'AU', 90 | 'doc_state': None, 91 | } 92 | res = service.post('/v1/ids', data, auth=token(user)) 93 | assert error(res, ObjectNotFound) 94 | 95 | 96 | def test_invalid_state(service, user, token, upload): 97 | upload1 = upload(user) 98 | upload2 = upload(user) 99 | 100 | data = { 101 | 'upload1': str(upload1.id), 102 | 'upload2': str(upload2.id), 103 | 'doc_type': 'PP', 104 | 'doc_country': 'AU', 105 | 'doc_state': '', 106 | } 107 | res = service.post('/v1/ids', data, auth=token(user)) 108 | assert error(res, InvalidState) 109 | 110 | 111 | def test_verify_package(service, user, token, ids_upload): 112 | ids = ids_upload(user) 113 | 114 | user.transition(ID_NOT_VERIFIED) 115 | 116 | with patch('upload.models.Upload.to_base64', return_value='blop'): 117 | res = service.post(f'/v1/ids/{ids.id}/verify', auth=token(user)) 118 | assert res.status_code == 200, res.json 119 | 120 | 121 | def test_verify_invalid_image(service, user, token, upload): 122 | upload1 = upload(user) 123 | upload2 = upload(user) 124 | user.transition(INFO_VERIFIED) 125 | data = { 126 | 'upload1': str(upload1.id), 127 | 'upload2': str(upload2.id), 128 | 'doc_type': 'PP', 129 | 'doc_country': 'AU', 130 | 'doc_state': '', 131 | } 132 | 133 | with patch('upload.models.Upload.stored_size', 100): 134 | res = service.post('/v1/ids', data, auth=token(user)) 135 | assert error(res, ValidationError) 136 | -------------------------------------------------------------------------------- /app/ids/views.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask import Blueprint, jsonify, request 3 | from voluptuous import All, Any, Coerce, In, Length, Optional, REMOVE_EXTRA 4 | 5 | from errors import ValidationError 6 | from ids.const import DOC_TYPES 7 | from ids.models import IDUpload 8 | from schema import Schema 9 | from upload.errors import MissingFile 10 | from upload.models import Upload 11 | from user.auth import authenticate 12 | from user.errors import InvalidState 13 | from user.models import User 14 | from user.state import ID_NOT_VERIFIED, INFO_VERIFIED 15 | from user.verifications import verify_ids 16 | 17 | blueprint = Blueprint('ids', __name__) 18 | 19 | 20 | @blueprint.route('/v1/ids', methods=['POST']) 21 | @authenticate() 22 | def submit_ids(user: User): 23 | """ 24 | Route to create ids upload package. 25 | 26 | User must be in `INFO_VERIFIED` state. 27 | Will transition user to `ID_NOT_VERIFIED` 28 | 29 | One should call `/v1/ids//verify` in order to actually start verification process (see docs there) 30 | """ 31 | if user.state != INFO_VERIFIED: 32 | raise InvalidState(user.state) 33 | 34 | schema = Schema({ 35 | 'upload1': Coerce(ObjectId), 36 | 'upload2': Coerce(ObjectId), 37 | 'doc_type': In(DOC_TYPES), 38 | 'doc_country': All(Length(2, 2), str), 39 | Optional('doc_state', default=None): Any(None, All(Length(2, 20), str)), 40 | }, extra=REMOVE_EXTRA, required=True) 41 | data = schema(request.json) 42 | upload1 = Upload.objects(user=user, id=data['upload1']).get() 43 | upload2 = Upload.objects(user=user, id=data['upload2']).get() 44 | 45 | try: 46 | if not (upload1.stored_size <= 4 * 1024 * 1024): 47 | raise ValidationError('Invalid image size') 48 | 49 | if not (upload2.stored_size <= 4 * 1024 * 1024): 50 | raise ValidationError('Invalid image size') 51 | except KeyError as err: 52 | raise MissingFile(str(err)) 53 | 54 | id_upload = IDUpload.create( 55 | user=user, 56 | upload1=upload1, 57 | upload2=upload2, 58 | doc_country=data['doc_country'], 59 | doc_state=data['doc_state'], 60 | doc_type=data['doc_type'], 61 | ) 62 | 63 | user.update(doc_type=data['doc_type']) 64 | user.transition(ID_NOT_VERIFIED) 65 | 66 | return jsonify(id_upload.to_json()), 201 67 | 68 | 69 | @blueprint.route('/v1/ids//verify', methods=['POST']) 70 | @authenticate() 71 | def verify_ids_package(id: str, user: User): 72 | """ 73 | Route that does id package verification. This can be a lengthy call, so it's moved into a separate one. 74 | 75 | Expects user to be in `ID_NOT_VERIFIED` state. 76 | Will transition user to `ID_VERIFIED` on success. 77 | 78 | Returns result of verification in `status` key 79 | """ 80 | if user.state != ID_NOT_VERIFIED: 81 | raise InvalidState(user.state) 82 | 83 | try: 84 | upload_id = ObjectId(id) 85 | except ValueError: 86 | raise ValidationError() 87 | 88 | upload = IDUpload.objects(user=user, id=upload_id).get() 89 | 90 | verify_ids(upload) 91 | 92 | return jsonify(user.reload().to_json()) 93 | -------------------------------------------------------------------------------- /app/onfid/__init__.py: -------------------------------------------------------------------------------- 1 | import onfid.signals 2 | -------------------------------------------------------------------------------- /app/onfid/api.py: -------------------------------------------------------------------------------- 1 | from onfido import Api 2 | 3 | from config import ONFIDO_TOKEN 4 | from session import build_session 5 | 6 | api = Api(ONFIDO_TOKEN) 7 | 8 | session = build_session(1024) 9 | 10 | 11 | def get_href(href: str) -> dict: 12 | res = session.get( 13 | href if 'https://' in href else f'https://api.onfido.com{href}', 14 | headers={'Authorization': f'Token token={ONFIDO_TOKEN}', 'Accept': 'application/json'}, 15 | ) 16 | return res.json() 17 | -------------------------------------------------------------------------------- /app/onfid/const.py: -------------------------------------------------------------------------------- 1 | CHECK_IN_PROGRESS = 'in_progress' 2 | CHECK_AWAITING_APPLICANT = 'awaiting_applicant' 3 | CHECK_COMPLETE = 'complete' 4 | CHECK_WITHDRAWN = 'withdrawn' 5 | CHECK_PAUSED = 'paused' 6 | CHECK_REOPENED = 'reopened' 7 | 8 | REPORT_AWAITING_DATA = 'awaiting_data' 9 | REPORT_AWAITING_APPROVAL = 'awaiting_approval' 10 | REPORT_COMPLETE = 'complete' 11 | REPORT_WITHDRAWN = 'withdrawn' 12 | REPORT_PAUSED = 'paused' 13 | REPORT_CANCELLED = 'cancelled' 14 | 15 | # If the report has returned information that needs to be evaluated, the overall result will be consider. 16 | # If some of the reports contained in the check have either consider or unidentified as their results. 17 | RESULT_CONSIDER = 'consider' 18 | 19 | RESULT_CLEAR = 'clear' #: If all the reports contained in the check have clear as their results. 20 | 21 | # Identity report (standard variant) only - this is returned if the applicant fails an identity check. This indicates 22 | # there is no identity match for this applicant on any of the databases searched. 23 | RESULT_UNIDENTIFIED = 'unidentified' 24 | 25 | # Subresults 26 | 27 | SUBRESULT_CLEAR = 'clear' #: If all underlying verifications pass, the overall sub result will be clear 28 | 29 | # If the report has returned information where the check cannot be processed further (poor quality image or an 30 | # unsupported document). 31 | SUBRESULT_REJECTED = 'rejected' 32 | 33 | SUBRESULT_SUSPECTED = 'suspected' #: If the document that is analysed is suspected to be fraudulent. 34 | 35 | # If any other underlying verifications fail but they don’t necessarily point to a fraudulent document (such as the 36 | # name provided by the applicant doesn’t match the one on the document) 37 | SUBRESULT_CAUTION = 'caution' 38 | -------------------------------------------------------------------------------- /app/onfid/geo.py: -------------------------------------------------------------------------------- 1 | iso2_iso3 = { 2 | 'BD': 'BGD', 'BE': 'BEL', 'BF': 'BFA', 'BG': 'BGR', 'BA': 'BIH', 'BB': 'BRB', 3 | 'WF': 'WLF', 'BL': 'BLM', 'BM': 'BMU', 'BN': 'BRN', 'BO': 'BOL', 'BH': 'BHR', 4 | 'BI': 'BDI', 'BJ': 'BEN', 'BT': 'BTN', 'JM': 'JAM', 'BV': 'BVT', 'BW': 'BWA', 5 | 'WS': 'WSM', 'BQ': 'BES', 'BR': 'BRA', 'BS': 'BHS', 'JE': 'JEY', 'BY': 'BLR', 6 | 'BZ': 'BLZ', 'RU': 'RUS', 'RW': 'RWA', 'RS': 'SRB', 'TL': 'TLS', 'RE': 'REU', 7 | 'TM': 'TKM', 'TJ': 'TJK', 'RO': 'ROU', 'TK': 'TKL', 'GW': 'GNB', 'GU': 'GUM', 8 | 'GT': 'GTM', 'GS': 'SGS', 'GR': 'GRC', 'GQ': 'GNQ', 'GP': 'GLP', 'JP': 'JPN', 9 | 'GY': 'GUY', 'GG': 'GGY', 'GF': 'GUF', 'GE': 'GEO', 'GD': 'GRD', 'GB': 'GBR', 10 | 'GA': 'GAB', 'GN': 'GIN', 'GM': 'GMB', 'GL': 'GRL', 'GI': 'GIB', 'GH': 'GHA', 11 | 'OM': 'OMN', 'TN': 'TUN', 'JO': 'JOR', 'HR': 'HRV', 'HT': 'HTI', 'HU': 'HUN', 12 | 'HK': 'HKG', 'HN': 'HND', 'HM': 'HMD', 'VE': 'VEN', 'PR': 'PRI', 'PS': 'PSE', 13 | 'PW': 'PLW', 'PT': 'PRT', 'KN': 'KNA', 'PY': 'PRY', 'IQ': 'IRQ', 'PA': 'PAN', 14 | 'PF': 'PYF', 'PG': 'PNG', 'PE': 'PER', 'PK': 'PAK', 'PH': 'PHL', 'PN': 'PCN', 15 | 'PL': 'POL', 'PM': 'SPM', 'ZM': 'ZMB', 'EH': 'ESH', 'EE': 'EST', 'EG': 'EGY', 16 | 'ZA': 'ZAF', 'EC': 'ECU', 'IT': 'ITA', 'VN': 'VNM', 'SB': 'SLB', 'ET': 'ETH', 17 | 'SO': 'SOM', 'ZW': 'ZWE', 'SA': 'SAU', 'ES': 'ESP', 'ER': 'ERI', 'ME': 'MNE', 18 | 'MD': 'MDA', 'MG': 'MDG', 'MF': 'MAF', 'MA': 'MAR', 'MC': 'MCO', 'UZ': 'UZB', 19 | 'MM': 'MMR', 'ML': 'MLI', 'MO': 'MAC', 'MN': 'MNG', 'MH': 'MHL', 'MK': 'MKD', 20 | 'MU': 'MUS', 'MT': 'MLT', 'MW': 'MWI', 'MV': 'MDV', 'MQ': 'MTQ', 'MP': 'MNP', 21 | 'MS': 'MSR', 'MR': 'MRT', 'IM': 'IMN', 'UG': 'UGA', 'TZ': 'TZA', 'MY': 'MYS', 22 | 'MX': 'MEX', 'IL': 'ISR', 'FR': 'FRA', 'AW': 'ABW', 'SH': 'SHN', 'SJ': 'SJM', 23 | 'FI': 'FIN', 'FJ': 'FJI', 'FK': 'FLK', 'FM': 'FSM', 'FO': 'FRO', 'NI': 'NIC', 24 | 'NL': 'NLD', 'NO': 'NOR', 'NA': 'NAM', 'VU': 'VUT', 'NC': 'NCL', 'NE': 'NER', 25 | 'NF': 'NFK', 'NG': 'NGA', 'NZ': 'NZL', 'NP': 'NPL', 'NR': 'NRU', 'NU': 'NIU', 26 | 'CK': 'COK', 'CI': 'CIV', 'CH': 'CHE', 'CO': 'COL', 'CN': 'CHN', 'CM': 'CMR', 27 | 'CL': 'CHL', 'CC': 'CCK', 'CA': 'CAN', 'CG': 'COG', 'CF': 'CAF', 'CD': 'COD', 28 | 'CZ': 'CZE', 'CY': 'CYP', 'CX': 'CXR', 'CR': 'CRI', 'CW': 'CUW', 'CV': 'CPV', 29 | 'CU': 'CUB', 'SZ': 'SWZ', 'SY': 'SYR', 'SX': 'SXM', 'KG': 'KGZ', 'KE': 'KEN', 30 | 'SS': 'SSD', 'SR': 'SUR', 'KI': 'KIR', 'KH': 'KHM', 'SV': 'SLV', 'KM': 'COM', 31 | 'ST': 'STP', 'SK': 'SVK', 'KR': 'KOR', 'SI': 'SVN', 'KP': 'PRK', 'KW': 'KWT', 32 | 'SN': 'SEN', 'SM': 'SMR', 'SL': 'SLE', 'SC': 'SYC', 'KZ': 'KAZ', 'KY': 'CYM', 33 | 'SG': 'SGP', 'SE': 'SWE', 'SD': 'SDN', 'DO': 'DOM', 'DM': 'DMA', 'DJ': 'DJI', 34 | 'DK': 'DNK', 'VG': 'VGB', 'DE': 'DEU', 'YE': 'YEM', 'DZ': 'DZA', 'US': 'USA', 35 | 'UY': 'URY', 'YT': 'MYT', 'UM': 'UMI', 'LB': 'LBN', 'LC': 'LCA', 'LA': 'LAO', 36 | 'TV': 'TUV', 'TW': 'TWN', 'TT': 'TTO', 'TR': 'TUR', 'LK': 'LKA', 'LI': 'LIE', 37 | 'LV': 'LVA', 'TO': 'TON', 'LT': 'LTU', 'LU': 'LUX', 'LR': 'LBR', 'LS': 'LSO', 38 | 'TH': 'THA', 'TF': 'ATF', 'TG': 'TGO', 'TD': 'TCD', 'TC': 'TCA', 'LY': 'LBY', 39 | 'VA': 'VAT', 'VC': 'VCT', 'AE': 'ARE', 'AD': 'AND', 'AG': 'ATG', 'AF': 'AFG', 40 | 'AI': 'AIA', 'VI': 'VIR', 'IS': 'ISL', 'IR': 'IRN', 'AM': 'ARM', 'AL': 'ALB', 41 | 'AO': 'AGO', 'AQ': 'ATA', 'AS': 'ASM', 'AR': 'ARG', 'AU': 'AUS', 'AT': 'AUT', 42 | 'IO': 'IOT', 'IN': 'IND', 'AX': 'ALA', 'AZ': 'AZE', 'IE': 'IRL', 'ID': 'IDN', 43 | 'UA': 'UKR', 'QA': 'QAT', 'MZ': 'MOZ' 44 | } 45 | -------------------------------------------------------------------------------- /app/onfid/helpers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import mimetypes 3 | import os.path 4 | import time 5 | from typing import Callable, Optional 6 | 7 | import requests 8 | from onfido import DocumentType 9 | 10 | from config import ONFIDO_TOKEN 11 | from ids.const import DRIVING_LICENCE, ID_CARD, PASSPORT, RESIDENCE_PERMIT, UTILITY_BILL 12 | from onfid.api import api 13 | from onfid.geo import iso2_iso3 14 | from upload.models import Upload 15 | from user.models import User 16 | 17 | mimetypes.init() 18 | log = logging.getLogger(__name__) 19 | 20 | DOC_TYPES = { 21 | DRIVING_LICENCE: DocumentType.DrivingLicense, 22 | PASSPORT: DocumentType.Passport, 23 | ID_CARD: DocumentType.NationalIdentityCard, 24 | RESIDENCE_PERMIT: DocumentType.WorkPermit, 25 | UTILITY_BILL: DocumentType.Unknown, # We don't have that any way 26 | } 27 | 28 | 29 | def rate_limit(max_per_minute: int) -> Callable: 30 | interval = 60.0 / float(max_per_minute) 31 | 32 | def decorate(func): 33 | last_time_called = [0.0] 34 | 35 | def limited_function(*args, **kargs): 36 | elapsed = time.clock() - last_time_called[0] 37 | left_to_wait = interval - elapsed 38 | 39 | if left_to_wait > 0: 40 | time.sleep(left_to_wait) 41 | 42 | ret = func(*args, **kargs) 43 | last_time_called[0] = time.clock() 44 | 45 | return ret 46 | 47 | return limited_function 48 | 49 | return decorate 50 | 51 | 52 | def create_user(user: User) -> Optional[dict]: 53 | try: 54 | # log.debug('Creating user %s', user.id) 55 | created = api.Applicants.create({ 56 | 'first_name': user.first_name, 57 | 'last_name': user.last_name, 58 | 'email': user.email, 59 | 'dob': user.dob.date().isoformat(), 60 | 'telephone': user.phone, # 'mobile' ? 61 | 'country': iso2_iso3.get(user.country_code, user.country_code), 62 | 'addresses': [ 63 | { 64 | 'street': user.address[:32], 65 | 'town': user.city, 66 | 'state': user.state_code, 67 | 'postcode': user.zip_code, 68 | 'country': iso2_iso3.get(user.country_code, user.country_code), 69 | }, 70 | ], 71 | }) 72 | # log.info('Created user: %s', created) 73 | log.info('Onfid id for user %s is %s', user.id, created.get('id', created)) 74 | user.modify(onfid_id=created.get('id')) 75 | return created 76 | except Exception as ex: 77 | log.exception('Onfido create user error: %s', ex) 78 | return None 79 | 80 | 81 | def custom_upload(applicant_id: str, document, document_filename: str, doc_type: str, face: bool) -> dict: 82 | filename, extension = os.path.splitext(document_filename) 83 | 84 | return api.Documents.post( 85 | "applicants/{0}/documents/".format(applicant_id), 86 | {"type": doc_type, 'side': 'front' if face else 'back'}, 87 | {"file": (document_filename, document, mimetypes.types_map[extension])} 88 | ) 89 | 90 | 91 | def upload_file(upload: Upload, doc_type: str, face: bool) -> dict: 92 | # log.debug('Uploading file %s/%s', doc_type, upload.id) 93 | res = custom_upload( 94 | str(upload.user.onfid_id), 95 | upload.fh, 96 | f'{upload.id}.{upload.extension}'.lower(), 97 | DOC_TYPES[doc_type], 98 | face, 99 | 100 | ) 101 | log.info('Created document: %s/%s as %s', doc_type, upload.id, res.get('id', res)) 102 | return res 103 | 104 | 105 | def register_webhook(env: str, base_url: str = None) -> dict: 106 | assert env in ['live', 'sandbox'], 'Env can be "live" or "sandbox"' 107 | 108 | res = requests.post( 109 | 'https://api.onfido.com/v2/webhooks', 110 | headers={'Authorization': f'Token token={ONFIDO_TOKEN}'}, 111 | json={ 112 | 'url': base_url or f'https://whitelist.dock.io/api/v1/onfido/{ONFIDO_TOKEN}', 113 | 'enabled': True, 114 | 'environments': [env], 115 | 116 | } 117 | ) 118 | return res.json() 119 | -------------------------------------------------------------------------------- /app/onfid/models.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | 4 | from blinker import Signal 5 | from mongoengine import DateTimeField, DictField, Document, ListField, ReferenceField, StringField 6 | 7 | from onfid.api import api, get_href 8 | from user.models import User 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | class Report(Document): 14 | meta = { 15 | 'indexes': ['status', 'result', 'name'] 16 | } 17 | 18 | id = StringField(primary_key=True) 19 | status = StringField() 20 | name = StringField() 21 | result = StringField() 22 | sub_result = StringField() 23 | raw = DictField() 24 | 25 | on_update = Signal() 26 | 27 | @classmethod 28 | def from_response(cls, response: dict) -> 'Report': 29 | values = dict( 30 | id=response.get('id'), 31 | status=response.get('status'), 32 | name=response.get('name'), 33 | result=response.get('result'), 34 | sub_result=response.get('sub_result'), 35 | raw=response, 36 | ) 37 | try: 38 | obj = cls(**values).save(force_insert=True) 39 | except: 40 | obj = cls.objects(id=values['id']).get() 41 | obj.modify(**values) 42 | cls.on_update.send(obj) 43 | return obj 44 | 45 | @classmethod 46 | def from_href(cls, href: str) -> 'Report': 47 | return cls.from_response(get_href(href)) 48 | 49 | 50 | class Check(Document): 51 | meta = { 52 | 'indexes': ['user', 'status', 'result'] 53 | } 54 | id = StringField(primary_key=True) 55 | user = ReferenceField(User) 56 | status = StringField() 57 | name = StringField() 58 | result = StringField() 59 | sub_result = StringField() 60 | raw = DictField() 61 | 62 | reports = ListField(StringField()) 63 | 64 | on_update = Signal() 65 | 66 | @classmethod 67 | def from_response(cls, response: dict, user: User = None) -> 'Check': 68 | reports = response.get('reports') 69 | values = dict( 70 | id=response.get('id'), 71 | status=response.get('status'), 72 | name=response.get('name'), 73 | result=response.get('result'), 74 | sub_result=response.get('sub_result'), 75 | reports=[itm.get('id') if isinstance(itm, dict) else itm for itm in reports] if reports else [], 76 | raw=response, 77 | ) 78 | 79 | if user: 80 | values.update(user=user) 81 | 82 | try: 83 | obj = cls(**values).save(force_insert=True) 84 | except: 85 | obj = cls.objects(id=values['id']).get() 86 | obj.modify(**values) 87 | cls.on_update.send(obj) 88 | return obj 89 | 90 | @classmethod 91 | def from_href(cls, href: str) -> 'Check': 92 | return cls.from_response(get_href(href)) 93 | 94 | @classmethod 95 | def create(cls, user: User) -> 'Check': 96 | data = { 97 | 'type': 'express', 98 | 'reports': [ 99 | {'name': 'watchlist', 'variant': 'full'}, 100 | {'name': 'document'}, 101 | ], 102 | 'async': True, 103 | } 104 | check = api.Checks.create(str(user.onfid_id), data) 105 | log.info('Created check: %s', check.get('id', check)) 106 | return cls.from_response(check, user=user) 107 | 108 | 109 | class Webhook(Document): 110 | CHECK = 'check' 111 | REPORT = 'report' 112 | 113 | resource_type = StringField() 114 | action = StringField() 115 | obj = DictField() 116 | 117 | response_data = DictField() 118 | received_at = DateTimeField(default=datetime.datetime.utcnow) 119 | 120 | on_create = Signal() 121 | 122 | @classmethod 123 | def from_response(cls, response: dict) -> 'Webhook': 124 | obj = cls( 125 | resource_type=response.get('resource_type'), 126 | action=response.get('action'), 127 | obj=response.get('object'), 128 | response_data=response, 129 | ).save() 130 | cls.on_create.send(obj) 131 | return obj 132 | -------------------------------------------------------------------------------- /app/onfid/signals.py: -------------------------------------------------------------------------------- 1 | from onfid.const import CHECK_COMPLETE 2 | from onfid.models import Check 3 | from signals import connect 4 | 5 | 6 | @connect(Check.on_update) 7 | def update_user_property_on_complete(check: Check, **_): 8 | if check.status != CHECK_COMPLETE: 9 | return 10 | 11 | check.user.modify(onfid_status=check.result) 12 | -------------------------------------------------------------------------------- /app/onfid/views.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from flask import Blueprint, jsonify, request 4 | from werkzeug.routing import ValidationError 5 | 6 | from config import ONFIDO_TOKEN 7 | from onfid.models import Check, Report, Webhook 8 | 9 | log = logging.getLogger(__name__) 10 | blueprint = Blueprint('onfido', __name__) 11 | 12 | 13 | @blueprint.route(f'/v1/onfido/{ONFIDO_TOKEN}', methods=['POST']) 14 | def onfido_webhook(): 15 | json = request.get_json(force=True) 16 | log.info(json) 17 | 18 | response = Webhook.from_response(json.get('payload')) 19 | if response.resource_type == Webhook.CHECK: 20 | cls = Check 21 | elif response.resource_type == Webhook.REPORT: 22 | cls = Report 23 | else: 24 | log.error('Invalid webhook type: %s', response.resource_type) 25 | raise ValidationError() 26 | 27 | cls.from_href(response.obj.get('href')) 28 | 29 | return jsonify({'status': 'ok'}) 30 | -------------------------------------------------------------------------------- /app/pylint.cfg: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Profiled execution. 11 | profile=no 12 | 13 | # Add files or directories to the blacklist. They should be base names, not 14 | # paths. 15 | ignore=CVS,git 16 | 17 | # Pickle collected data for later comparisons. 18 | persistent=no 19 | 20 | # List of plugins (as comma separated values of python modules names) to load, 21 | # usually to register additional checkers. 22 | load-plugins= 23 | 24 | # Deprecated. It was used to include message's id in output. Use --msg-template 25 | # instead. 26 | #include-ids=no 27 | 28 | # Deprecated. It was used to include symbolic ids of messages in output. Use 29 | # --msg-template instead. 30 | #symbols=no 31 | 32 | # Use multiple processes to speed up Pylint. 33 | jobs=2 34 | 35 | # Allow loading of arbitrary C extensions. Extensions are imported into the 36 | # active Python interpreter and may run arbitrary code. 37 | unsafe-load-any-extension=no 38 | 39 | # A comma-separated list of package or module names from where C extensions may 40 | # be loaded. Extensions are loading into the active Python interpreter and may 41 | # run arbitrary code 42 | extension-pkg-whitelist= 43 | 44 | # Allow optimization of some AST trees. This will activate a peephole AST 45 | # optimizer, which will apply various small optimizations. For instance, it can 46 | # be used to obtain the result of joining multiple strings with the addition 47 | # operator. Joining a lot of strings can lead to a maximum recursion error in 48 | # Pylint and this flag can prevent that. It has one side effect, the resulting 49 | # AST will be different than the one from reality. 50 | optimize-ast=no 51 | 52 | 53 | [MESSAGES CONTROL] 54 | 55 | # Only show warnings with the listed confidence levels. Leave empty to show 56 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 57 | confidence= 58 | 59 | # Enable the message, report, category or checker with the given id(s). You can 60 | # either give multiple identifier separated by comma (,) or put this option 61 | # multiple time. See also the "--disable" option for examples. 62 | #enable= 63 | 64 | # Disable the message, report, category or checker with the given id(s). You 65 | # can either give multiple identifiers separated by comma (,) or put this 66 | # option multiple times (only on the command line, not in the configuration 67 | # file where it should appear only once).You can also use "--disable=all" to 68 | # disable everything first and then reenable specific checks. For example, if 69 | # you want to run only the similarities checker, you can use "--disable=all 70 | # --enable=similarities". If you want to run only the classes checker, but have 71 | # no Warning level messages displayed, use"--disable=all --enable=classes 72 | # --disable=W" 73 | disable=E1608,W1627,E1601,E1603,E1602,E1605,E1604,E1607,E1606,W1621,W1620,W1623,W1622,W1625,W1624,W1609,W1608,W1607,W1606,W1605,W1604,W1603,W1602,W1601,W1639,W1640,I0021,W1638,I0020,W1618,W1619,W1630,W1626,W1637,W1634,W1635,W1610,W1611,W1612,W1613,W1614,W1615,W1616,W1617,W1632,W1633,W0704,W1628,W1629,W1636,C0111,E1101,W0621 74 | 75 | 76 | [REPORTS] 77 | 78 | # Set the output format. Available formats are text, parseable, colorized, msvs 79 | # (visual studio) and html. You can also give a reporter class, eg 80 | # mypackage.mymodule.MyReporterClass. 81 | output-format=parseable 82 | 83 | # Put messages in a separate file for each module / package specified on the 84 | # command line instead of printing them on stdout. Reports (if any) will be 85 | # written in a file name "pylint_global.[txt|html]". 86 | files-output=no 87 | 88 | # Tells whether to display a full report or only the messages 89 | reports=yes 90 | 91 | # Python expression which should return a note less than 10 (10 is the highest 92 | # note). You have access to the variables errors warning, statement which 93 | # respectively contain the number of errors / warnings messages and the total 94 | # number of statements analyzed. This is used by the global evaluation report 95 | # (RP0004). 96 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 97 | 98 | # Add a comment according to your evaluation note. This is used by the global 99 | # evaluation report (RP0004). 100 | comment=no 101 | 102 | # Template used to display messages. This is a python new-style format string 103 | # used to format the message information. See doc for all details 104 | #msg-template= 105 | 106 | 107 | [LOGGING] 108 | 109 | # Logging modules to check that the string format arguments are in logging 110 | # function parameter format 111 | logging-modules=logging 112 | 113 | 114 | [FORMAT] 115 | 116 | # Maximum number of characters on a single line. 117 | max-line-length=120 118 | 119 | # Regexp for a line that is allowed to be longer than the limit. 120 | ignore-long-lines=^\s*(# )??$ 121 | 122 | # Allow the body of an if to be on the same line as the test if there is no 123 | # else. 124 | single-line-if-stmt=no 125 | 126 | # List of optional constructs for which whitespace checking is disabled 127 | no-space-check=trailing-comma,dict-separator 128 | 129 | # Maximum number of lines in a module 130 | max-module-lines=1000 131 | 132 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 133 | # tab). 134 | indent-string=' ' 135 | 136 | # Number of spaces of indent required inside a hanging or continued line. 137 | indent-after-paren=4 138 | 139 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 140 | expected-line-ending-format= 141 | 142 | 143 | [VARIABLES] 144 | 145 | # Tells whether we should check for unused import in __init__ files. 146 | init-import=no 147 | 148 | # A regular expression matching the name of dummy variables (i.e. expectedly 149 | # not used). 150 | dummy-variables-rgx=_$|dummy 151 | 152 | # List of additional names supposed to be defined in builtins. Remember that 153 | # you should avoid to define new builtins when possible. 154 | additional-builtins= 155 | 156 | # List of strings which can identify a callback function by name. A callback 157 | # name must start or end with one of those strings. 158 | callbacks=cb_,_cb 159 | 160 | 161 | [BASIC] 162 | 163 | # Required attributes for module, separated by a comma 164 | required-attributes= 165 | 166 | # List of builtins function names that should not be used, separated by a comma 167 | bad-functions=map,filter,input 168 | 169 | # Good variable names which should always be accepted, separated by a comma 170 | good-names=i,j,k,ex,Run,_ 171 | 172 | # Bad variable names which should always be refused, separated by a comma 173 | bad-names=foo,bar,baz,toto,tutu,tata 174 | 175 | # Colon-delimited sets of names that determine each other's naming style when 176 | # the name regexes allow several styles. 177 | name-group= 178 | 179 | # Include a hint for the correct naming format with invalid-name 180 | include-naming-hint=yes 181 | 182 | # Regular expression matching correct function names 183 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 184 | 185 | # Naming hint for function names 186 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 187 | 188 | # Regular expression matching correct variable names 189 | variable-rgx=[a-z_][a-z0-9_]{1,30}$ 190 | 191 | # Naming hint for variable names 192 | variable-name-hint=[a-z_][a-z0-9_]{1,30}$ 193 | 194 | # Regular expression matching correct constant names 195 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 196 | 197 | # Naming hint for constant names 198 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 199 | 200 | # Regular expression matching correct attribute names 201 | attr-rgx=[a-z_][a-z0-9_]{1,30}$ 202 | 203 | # Naming hint for attribute names 204 | attr-name-hint=[a-z_][a-z0-9_]{1,30}$ 205 | 206 | # Regular expression matching correct argument names 207 | argument-rgx=[a-z_][a-z0-9_]{1,30}$ 208 | 209 | # Naming hint for argument names 210 | argument-name-hint=[a-z_][a-z0-9_]{1,30}$ 211 | 212 | # Regular expression matching correct class attribute names 213 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 214 | 215 | # Naming hint for class attribute names 216 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 217 | 218 | # Regular expression matching correct inline iteration names 219 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 220 | 221 | # Naming hint for inline iteration names 222 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 223 | 224 | # Regular expression matching correct class names 225 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 226 | 227 | # Naming hint for class names 228 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 229 | 230 | # Regular expression matching correct module names 231 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 232 | 233 | # Naming hint for module names 234 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 235 | 236 | # Regular expression matching correct method names 237 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 238 | 239 | # Naming hint for method names 240 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 241 | 242 | # Regular expression which should only match function or class names that do 243 | # not require a docstring. 244 | no-docstring-rgx=__.*__ 245 | 246 | # Minimum line length for functions/classes that require docstrings, shorter 247 | # ones are exempt. 248 | docstring-min-length=-1 249 | 250 | 251 | [MISCELLANEOUS] 252 | 253 | # List of note tags to take in consideration, separated by a comma. 254 | notes=FIXME,XXX,TODO 255 | 256 | 257 | [TYPECHECK] 258 | 259 | # Tells whether missing members accessed in mixin class should be ignored. A 260 | # mixin class is detected if its name ends with "mixin" (case insensitive). 261 | ignore-mixin-members=yes 262 | 263 | # List of module names for which member attributes should not be checked 264 | # (useful for modules/projects where namespaces are manipulated during runtime 265 | # and thus existing member attributes cannot be deduced by static analysis 266 | ignored-modules= 267 | 268 | # List of classes names for which member attributes should not be checked 269 | # (useful for classes with attributes dynamically set). 270 | ignored-classes=SQLObject 271 | 272 | # When zope mode is activated, add a predefined set of Zope acquired attributes 273 | # to generated-members. 274 | zope=no 275 | 276 | # List of members which are set dynamically and missed by pylint inference 277 | # system, and so shouldn't trigger E0201 when accessed. Python regular 278 | # expressions are accepted. 279 | generated-members=REQUEST,acl_users,aq_parent 280 | 281 | 282 | [SPELLING] 283 | 284 | # Spelling dictionary name. Available dictionaries: none. To make it working 285 | # install python-enchant package. 286 | spelling-dict= 287 | 288 | # List of comma separated words that should not be checked. 289 | spelling-ignore-words= 290 | 291 | # A path to a file that contains private dictionary; one word per line. 292 | spelling-private-dict-file= 293 | 294 | # Tells whether to store unknown words to indicated private dictionary in 295 | # --spelling-private-dict-file option instead of raising a message. 296 | spelling-store-unknown-words=no 297 | 298 | 299 | [SIMILARITIES] 300 | 301 | # Minimum lines number of a similarity. 302 | min-similarity-lines=4 303 | 304 | # Ignore comments when computing similarities. 305 | ignore-comments=yes 306 | 307 | # Ignore docstrings when computing similarities. 308 | ignore-docstrings=yes 309 | 310 | # Ignore imports when computing similarities. 311 | ignore-imports=no 312 | 313 | 314 | [CLASSES] 315 | 316 | # List of interface methods to ignore, separated by a comma. This is used for 317 | # instance to not check methods defines in Zope's Interface base class. 318 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 319 | 320 | # List of method names used to declare (i.e. assign) instance attributes. 321 | defining-attr-methods=__init__,__new__,setUp 322 | 323 | # List of valid names for the first argument in a class method. 324 | valid-classmethod-first-arg=cls 325 | 326 | # List of valid names for the first argument in a metaclass class method. 327 | valid-metaclass-classmethod-first-arg=mcs 328 | 329 | # List of member names, which should be excluded from the protected access 330 | # warning. 331 | exclude-protected=_asdict,_fields,_replace,_source,_make 332 | 333 | 334 | [IMPORTS] 335 | 336 | # Deprecated modules which should not be used, separated by a comma 337 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 338 | 339 | # Create a graph of every (i.e. internal and external) dependencies in the 340 | # given file (report RP0402 must not be disabled) 341 | import-graph= 342 | 343 | # Create a graph of external dependencies in the given file (report RP0402 must 344 | # not be disabled) 345 | ext-import-graph= 346 | 347 | # Create a graph of internal dependencies in the given file (report RP0402 must 348 | # not be disabled) 349 | int-import-graph= 350 | 351 | 352 | [DESIGN] 353 | 354 | # Maximum number of arguments for function / method 355 | max-args=5 356 | 357 | # Argument names that match this expression will be ignored. Default to name 358 | # with leading underscore 359 | ignored-argument-names=_.* 360 | 361 | # Maximum number of locals for function / method body 362 | max-locals=15 363 | 364 | # Maximum number of return / yield for function / method body 365 | max-returns=6 366 | 367 | # Maximum number of branch for function / method body 368 | max-branches=12 369 | 370 | # Maximum number of statements in function / method body 371 | max-statements=50 372 | 373 | # Maximum number of parents for a class (see R0901). 374 | max-parents=7 375 | 376 | # Maximum number of attributes for a class (see R0902). 377 | max-attributes=7 378 | 379 | # Minimum number of public methods for a class (see R0903). 380 | min-public-methods=2 381 | 382 | # Maximum number of public methods for a class (see R0904). 383 | max-public-methods=20 384 | 385 | 386 | [EXCEPTIONS] 387 | 388 | # Exceptions that will emit a warning when being caught. Defaults to 389 | # "Exception" 390 | overgeneral-exceptions=Exception 391 | -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | blinker==1.4 2 | flask==0.12.2 3 | gevent==1.2.2 4 | itsdangerous==0.24 5 | mongoengine==0.15.0 6 | mongomock==3.8.0 7 | flake8==3.5.0 8 | pylint==1.8.1 9 | pytest-flask==0.10.0 10 | pytest-xdist==1.20.1 11 | pytest==3.3.1 12 | raven[flask]==6.4.0 13 | requests==2.18.4 14 | teamcity-messages==1.21 15 | voluptuous==0.10.5 16 | uwsgi==2.0.15 17 | boto==2.48.0 18 | requests_mock==1.4.0 19 | python-telegram-bot==9.0.0 20 | flask-cors 21 | pysftp==0.2.9 22 | pyonfido -------------------------------------------------------------------------------- /app/routes.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from werkzeug.routing import BaseConverter, ValidationError 3 | 4 | import bot.views 5 | import eth.views 6 | import idm.views 7 | import ids.views 8 | import onfid.views 9 | import upload.views 10 | import user.views 11 | from tokens import get_token, verify_token 12 | 13 | 14 | def load_blueprints(): 15 | return [ 16 | eth.views.blueprint, 17 | ids.views.blueprint, 18 | idm.views.blueprint, 19 | onfid.views.blueprint, 20 | bot.views.blueprint, 21 | user.views.blueprint, 22 | upload.views.blueprint, 23 | ] 24 | 25 | 26 | class TokenConverter(BaseConverter): 27 | 28 | def to_python(self, value: str) -> ObjectId: 29 | try: 30 | return verify_token(value) 31 | except ValueError: 32 | raise ValidationError(value) 33 | 34 | def to_url(self, value: ObjectId) -> str: 35 | return get_token(value) 36 | -------------------------------------------------------------------------------- /app/schema.py: -------------------------------------------------------------------------------- 1 | from voluptuous import Schema, Optional, Coerce, MultipleInvalid, In, All, ALLOW_EXTRA, Any, Required # noqa 2 | 3 | _ = MultipleInvalid # Dummy line to Make PyCharm formatter happy 4 | -------------------------------------------------------------------------------- /app/sentry.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from flask import Flask 4 | from raven import Client, base 5 | from raven.conf import setup_logging as setup_sentry_logging 6 | from raven.contrib.flask import Sentry 7 | from raven.handlers.logging import SentryHandler 8 | from raven.transport.gevent import GeventedHTTPTransport 9 | 10 | 11 | class DummyClient(base.DummyClient): 12 | last_event_id = 'dummy' 13 | 14 | 15 | def setup_sentry(app: Flask): 16 | sentry_dsn = app.config.get('SENTRY_DSN') 17 | if not sentry_dsn: 18 | client = DummyClient() 19 | else: 20 | client = Client( 21 | sentry_dsn, 22 | transport=GeventedHTTPTransport, 23 | release=app.config.get('CURRENT_RELEASE'), 24 | environment=app.config.get('CURRENT_ENVIRONMENT'), 25 | ) 26 | 27 | sentry = Sentry(client=client, logging=True, level=logging.WARNING) 28 | handler = SentryHandler(client, level=logging.WARNING) 29 | setup_sentry_logging(handler) 30 | sentry.init_app(app) 31 | 32 | return sentry, client 33 | -------------------------------------------------------------------------------- /app/session.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import requests.adapters 3 | 4 | 5 | def build_session(pool_size: int = 100) -> requests.Session: 6 | session = requests.Session() 7 | adapter = requests.adapters.HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size) 8 | session.mount('http://', adapter) 9 | session.mount('https://', adapter) 10 | return session 11 | -------------------------------------------------------------------------------- /app/signals.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import logging 3 | from typing import Callable, Optional, Tuple 4 | 5 | import blinker 6 | 7 | from user.models import User 8 | 9 | log = logging.getLogger(__name__) 10 | Transition = Tuple[str, str] 11 | 12 | 13 | def connect(*signals, **kwargs): 14 | """ 15 | Decorator to connect method to several signals. 16 | 17 | :param sender: If set will make decorated method to be called only when particular sender has 18 | called it. Defaults to ``ANY`` 19 | 20 | :param weak: If true, the Signal will hold a weakref to *receiver* 21 | and automatically disconnect when *receiver* goes out of scope or 22 | is garbage collected. Defaults to ``True``. 23 | 24 | """ 25 | sender = kwargs.get('sender', blinker.ANY) 26 | weak = kwargs.get('weak', True) 27 | 28 | def decorator(func): 29 | 30 | for signal in signals: 31 | signal.connect(func, sender=sender, weak=weak) 32 | 33 | return func 34 | 35 | return decorator 36 | 37 | 38 | def transition(state_before: Optional[str], state_now: str) -> Callable: 39 | def wrapper(func): 40 | def handler(user: User, transition: Transition, **kwargs): 41 | 42 | if state_before and transition != (state_before, state_now): 43 | return 44 | elif not state_before and transition[1] != state_now: 45 | return 46 | 47 | return func(user, transition=transition, **kwargs) 48 | 49 | User.on_transition.connect(handler, weak=False) 50 | 51 | return wrapper 52 | 53 | 54 | def log_exception(func: Callable) -> Callable: 55 | @functools.wraps(func) 56 | def wrapper(*args, **kwargs): 57 | try: 58 | func(*args, **kwargs) 59 | except Exception as ex: 60 | log.exception('Unhandled signal exception: %s', ex) 61 | 62 | return wrapper 63 | -------------------------------------------------------------------------------- /app/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | SOMAXCONN=65535 4 | 5 | echo ${SOMAXCONN} 2>/dev/null > /proc/sys/net/core/somaxconn 6 | sysctl net.core.somaxconn=${SOMAXCONN} 2>/dev/null 7 | 8 | _term() { 9 | kill -HUP "$child" 2>/dev/null 10 | } 11 | 12 | trap _term TERM 13 | 14 | uwsgi uwsgi.ini & 15 | child=$! 16 | wait "$child" 17 | -------------------------------------------------------------------------------- /app/tokens/__init__.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from itsdangerous import BadSignature, URLSafeSerializer 3 | 4 | from config import SALT 5 | 6 | 7 | def signer() -> URLSafeSerializer: 8 | return URLSafeSerializer(SALT, salt=f'whitelist') 9 | 10 | 11 | def get_token(user_id: ObjectId) -> str: 12 | return signer().dumps([str(user_id)]) 13 | 14 | 15 | def verify_token(token: str) -> ObjectId: 16 | try: 17 | [user_id] = signer().loads(token) 18 | except BadSignature: 19 | raise ValueError() 20 | 21 | return ObjectId(user_id) 22 | -------------------------------------------------------------------------------- /app/tokens/test_token.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | 3 | from tokens import signer, verify_token 4 | 5 | ID = ObjectId('5a6dee770ee97c0001534b3e') 6 | WRITE_TOKEN = 'WyI1YTZkZWU3NzBlZTk3YzAwMDE1MzRiM2UiXQ.dWfypJfoIo6CD5aOSfrkgUVkKSs' 7 | 8 | 9 | def test_read_signer(): 10 | assert signer().dumps([str(ID)]) == WRITE_TOKEN 11 | assert signer().loads(WRITE_TOKEN) == [str(ID)] 12 | 13 | 14 | def test_validate_signer(): 15 | assert verify_token(WRITE_TOKEN) == ID 16 | -------------------------------------------------------------------------------- /app/upload/errors.py: -------------------------------------------------------------------------------- 1 | from errors import AppError 2 | 3 | 4 | class MissingFile(AppError): 5 | pass 6 | -------------------------------------------------------------------------------- /app/upload/models.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import datetime 3 | import logging 4 | import mimetypes 5 | 6 | from blinker import Signal 7 | from mongoengine import DateTimeField, Document, IntField, ReferenceField, StringField 8 | 9 | from upload.s3 import s3 10 | from user.models import User 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | PUT_URL_DEFAULT_EXPIRE = 60 * 60 * 24 * 7 15 | 16 | 17 | class Upload(Document): 18 | PUBLIC_FIELDS = ['id', 'content_type', 'size', 'original_filename', 'url'] 19 | 20 | meta = { 21 | 'indexes': ['user'], 22 | } 23 | 24 | user = ReferenceField(User, required=True) 25 | content_type = StringField() 26 | created_at = DateTimeField(default=datetime.datetime.utcnow) 27 | original_filename = StringField() 28 | size = IntField() 29 | url = StringField() 30 | put_url = StringField() 31 | 32 | on_create = Signal() 33 | 34 | @property 35 | def url(self): 36 | return s3.sign_url( 37 | filename=self.filename, 38 | method='GET', 39 | expire=60 * 60 * 24 * 365, # 1Y 40 | ) 41 | 42 | @property 43 | def filename(self): 44 | return f'{self.user.id}/{self.id}{"." if self.extension else ""}{self.extension}' 45 | 46 | @property 47 | def extension(self): 48 | if '.' in self.original_filename: 49 | return self.original_filename.rsplit('.', 1)[-1] 50 | else: 51 | return None 52 | 53 | @property 54 | def put_url(self): 55 | headers = { 56 | 'Content-Type': self.content_type, 57 | 'Content-Disposition': f'attachment; filename={self.filename}', 58 | } 59 | 60 | return s3.sign_url( 61 | filename=self.filename, 62 | headers=headers, 63 | method='PUT', 64 | expire=PUT_URL_DEFAULT_EXPIRE, 65 | ) 66 | 67 | @classmethod 68 | def create(cls, user: User, original_filename: str, content_type: str = None, size: int = None, **kwargs): 69 | if not content_type: 70 | content_type, _ = mimetypes.guess_type(original_filename) 71 | upload = cls( 72 | original_filename=original_filename, 73 | content_type=content_type, 74 | size=size, 75 | user=user, 76 | **kwargs 77 | ).save() 78 | cls.on_create.send(upload) 79 | return upload 80 | 81 | @staticmethod 82 | def upload(data: bytes, filename: str, content_type: str) -> str: 83 | headers = {'Content-Disposition': f'attachment; filename={filename}'} 84 | return s3.put( 85 | filename, 86 | data, 87 | content_type=content_type, 88 | **headers 89 | ) 90 | 91 | def to_base64(self) -> str: 92 | data = s3.get(self.filename) 93 | b64 = base64.b64encode(data).decode('ascii') 94 | return f'{self.content_type};base64,{b64}' 95 | 96 | @property 97 | def stored_size(self): 98 | """ Return size in bytes of the file stored on s3. This is more reliable than data provided by user. """ 99 | return s3.file_size(self.filename) 100 | 101 | @property 102 | def fh(self): 103 | return s3.key(self.filename) -------------------------------------------------------------------------------- /app/upload/s3.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from gzip import GzipFile 3 | from io import BytesIO 4 | 5 | import boto 6 | from boto.s3.connection import S3Connection 7 | from boto.s3.key import Key 8 | 9 | from config import AWS_ACCESS_KEY_ID, AWS_BUCKET, AWS_SECRET_ACCESS_KEY 10 | 11 | 12 | class S3(object): 13 | _connection = None 14 | 15 | @property 16 | def connection(self): 17 | if not self._connection and not AWS_ACCESS_KEY_ID: 18 | return 19 | 20 | if not self._connection: 21 | self._connection = S3Connection( 22 | AWS_ACCESS_KEY_ID, 23 | AWS_SECRET_ACCESS_KEY, 24 | calling_format=boto.s3.connection.OrdinaryCallingFormat() 25 | ) 26 | return self._connection 27 | 28 | def file_size(self, filename): 29 | bkt = self.connection.get_bucket(AWS_BUCKET, validate=False) 30 | key = bkt.lookup(filename) 31 | if not key: 32 | raise KeyError(filename) 33 | return key.size 34 | 35 | def key(self, filename): 36 | bkt = self.connection.get_bucket(AWS_BUCKET, validate=False) 37 | key = Key(bkt) 38 | key.key = filename 39 | return key 40 | 41 | def put(self, filename, data, content_type=None, **meta): 42 | if not self.connection: 43 | return None 44 | key = self.key(filename) 45 | if content_type: 46 | key.content_type = content_type 47 | for k, v in meta.items(): 48 | key.set_metadata(k, v) 49 | key.set_contents_from_string(data) 50 | return key 51 | 52 | def get(self, filename): 53 | if not self.connection: 54 | return None 55 | key = self.key(filename) 56 | return key.read() 57 | 58 | def delete(self, filename): 59 | if not self.connection: 60 | return None 61 | 62 | key = self.key(filename) 63 | if key.exists(): 64 | key.delete() 65 | return True 66 | return False 67 | 68 | def public_url(self, filename, bucket_as_domain=True, protocol='https'): 69 | if bucket_as_domain: 70 | return f'{protocol}://{AWS_BUCKET}/{filename}' 71 | else: 72 | return self.key(filename).generate_url(expires_in=0, query_auth=False) 73 | 74 | def sign_url(self, filename, method='PUT', expire=600, headers=None): 75 | if not self.connection: 76 | return None 77 | 78 | return self.connection.generate_url( 79 | bucket=AWS_BUCKET, 80 | expires_in=expire, 81 | force_http=False, 82 | headers=headers, 83 | key=filename, 84 | method=method, 85 | query_auth=True, 86 | ) 87 | 88 | def upload( 89 | self, 90 | filename, 91 | fileobj=None, 92 | gzip=False, # False or gzip compression level (0..9) 93 | content_type=None, 94 | signed_duration=36000, 95 | signed_method='GET', 96 | ): 97 | opened = False 98 | if not fileobj: 99 | fileobj = open(filename, 'r') 100 | opened = True 101 | 102 | try: 103 | if gzip in [None, False]: 104 | self.put( 105 | filename, 106 | fileobj.read(), 107 | content_type=content_type, 108 | **{'Content-Disposition': 'attachment; filename=' + filename} 109 | ) 110 | else: 111 | if gzip is True: 112 | gzip = 7 113 | iofh = BytesIO() 114 | try: 115 | with GzipFile(filename, 'wb', gzip, iofh) as gzfileobj: 116 | shutil.copyfileobj(fileobj, gzfileobj) 117 | 118 | filename = '{}.gz'.format(filename) 119 | content_type = 'application/gzip' 120 | self.put( 121 | filename, 122 | iofh.getvalue(), 123 | content_type=content_type, 124 | **{'Content-Disposition': 'attachment; filename=' + filename} 125 | ) 126 | finally: 127 | iofh.close() 128 | 129 | if signed_duration and signed_method: 130 | res = self.sign_url(filename, signed_method, signed_duration) 131 | return res 132 | 133 | return filename 134 | finally: 135 | if opened: 136 | fileobj.close() 137 | 138 | 139 | s3 = S3() 140 | -------------------------------------------------------------------------------- /app/upload/test_upload.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | 3 | from mock import patch 4 | 5 | from conftest import error 6 | from errors import ValidationError 7 | from upload.models import Upload 8 | 9 | 10 | def test_upload(service, user, token): 11 | data = { 12 | 'filename': 'blop.jpg', 13 | 'content_type': 'image/jpeg', 14 | 'size': 500 * 1024, 15 | } 16 | res = service.post('/v1/upload', data, auth=token(user)) 17 | assert res.status_code == 200, res.json 18 | 19 | upload = Upload.objects().get() 20 | assert upload.user == user 21 | assert upload.filename == f'{user.id}/{upload.id}.jpg' 22 | 23 | 24 | def test_base64(user): 25 | data = BytesIO(b'Hello') 26 | with patch('upload.s3.s3.get', return_value=data.read()): 27 | res = Upload(user=user, original_filename='blop', content_type='image/png').to_base64() 28 | assert res == 'image/png;base64,SGVsbG8=' 29 | 30 | 31 | def test_stored_size(user): 32 | with patch('upload.s3.S3.file_size', return_value=1234): 33 | res = Upload(user=user, original_filename='blop', content_type='image/png').stored_size 34 | assert res == 1234 35 | 36 | 37 | def test_no_extension(service, user, token): 38 | data = { 39 | 'filename': 'no_extension', 40 | 'content_type': 'image/jpeg', 41 | 'size': 500 * 1024, 42 | } 43 | res = service.post('/v1/upload', data, auth=token(user)) 44 | assert error(res, ValidationError) 45 | 46 | 47 | def test_invalid_extension(service, user, token): 48 | data = { 49 | 'filename': 'file.bMp', 50 | 'content_type': 'image/jpeg', 51 | 'size': 500 * 1024, 52 | } 53 | res = service.post('/v1/upload', data, auth=token(user)) 54 | assert error(res, ValidationError) 55 | -------------------------------------------------------------------------------- /app/upload/views.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, jsonify, request 2 | from voluptuous import All, Length, REMOVE_EXTRA, Range 3 | 4 | from errors import ValidationError 5 | from schema import Schema 6 | from upload.models import Upload 7 | from user.auth import authenticate 8 | from user.models import User 9 | 10 | blueprint = Blueprint('uploads', __name__) 11 | 12 | 13 | @blueprint.route('/v1/upload', methods=['POST']) 14 | @authenticate() 15 | def new_upload(user: User): 16 | schema = Schema({ 17 | 'filename': All(Length(3, 250), str), 18 | 'content_type': All(Length(5, 20), str), 19 | 'size': Range(100, 4 * 1024 * 1024), # 400KB..4MB 20 | }, extra=REMOVE_EXTRA, required=True) 21 | data = schema(request.json) 22 | 23 | try: 24 | ext = data['filename'].split('.')[1].lower() 25 | except: 26 | raise ValidationError('Invalid file') 27 | 28 | # if ext not in ['gif', 'jpeg', 'jpg', 'png']: 29 | # raise ValidationError('Invalid file') 30 | 31 | upload = Upload.create( 32 | user=user, 33 | original_filename=data['filename'], 34 | content_type=data['content_type'], 35 | size=data['size'], 36 | ) 37 | return jsonify({'put_url': upload.put_url, 'id': str(upload.id)}) 38 | -------------------------------------------------------------------------------- /app/user/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /app/user/auth.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import functools 3 | from typing import Callable 4 | 5 | from flask import request 6 | from mongoengine import DoesNotExist 7 | from werkzeug.exceptions import Unauthorized 8 | 9 | from config import WHITELIST_CLOSED, WHITELIST_OPEN_DATE 10 | from errors import WhitelistClosed 11 | from tokens import verify_token 12 | from user.errors import UserNotFound 13 | from user.models import User 14 | 15 | 16 | def authenticate(should_exist: bool = True, bypass_closing: bool = False) -> Callable: 17 | def wrapper(func: Callable) -> Callable: 18 | @functools.wraps(func) 19 | def inner(*args, **kwargs): 20 | if not bypass_closing and WHITELIST_CLOSED: 21 | raise WhitelistClosed() 22 | 23 | if not bypass_closing and WHITELIST_OPEN_DATE and WHITELIST_OPEN_DATE > datetime.datetime.utcnow(): 24 | raise WhitelistClosed(details=dict(open_ts=int(WHITELIST_OPEN_DATE.strftime('%s')))) 25 | 26 | auth = request.headers.get('Authorization') 27 | if not auth: 28 | raise Unauthorized() 29 | 30 | token = auth[6:] 31 | if not token: 32 | raise Unauthorized() 33 | 34 | try: 35 | user_id = verify_token(token) 36 | except ValueError: 37 | raise Unauthorized() 38 | 39 | try: 40 | user = User.objects(id=user_id).get() 41 | except DoesNotExist: 42 | if not should_exist: 43 | user = User(id=user_id) 44 | else: 45 | raise UserNotFound() 46 | 47 | return func(*args, user=user, **kwargs) 48 | 49 | return inner 50 | 51 | return wrapper 52 | -------------------------------------------------------------------------------- /app/user/errors.py: -------------------------------------------------------------------------------- 1 | from errors import AppError 2 | 3 | 4 | class UserError(AppError): 5 | pass 6 | 7 | 8 | class UserNotFound(UserError): 9 | code = 404 10 | 11 | 12 | class InvalidState(UserError): 13 | code = 406 14 | -------------------------------------------------------------------------------- /app/user/models.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from blinker import Signal 4 | from bson import ObjectId 5 | from mongoengine import BooleanField, DateTimeField, Document, EmailField, FloatField, IntField, StringField, ListField 6 | 7 | from tokens import get_token 8 | from user.errors import UserNotFound 9 | from user.state import ALL_DECLINE_REASONS, ALL_STATES, NEW_USER 10 | 11 | 12 | class User(Document): 13 | """ 14 | User object in whitelist service. 15 | 16 | """ 17 | meta = { 18 | 'indexes': ['state', 'kyc_result', 'idm_result', 'country_code', 'onfid_id', 'onfid_status'], 19 | } 20 | 21 | CSV_FIELDS = [ 22 | 'id', 23 | 'email', 24 | 'first_name', 25 | 'last_name', 26 | 'state', 27 | 'created_at', 28 | 'eth_address', 29 | 'eth_amount', 30 | 'phone', 31 | 'dob', 32 | 'telegram', 33 | 'doc_type', 34 | # 'confirmed_location', 35 | 'address', 36 | 'city', 37 | 'state_code', 38 | 'zip_code', 39 | 'country_code', 40 | 'ip_country', 41 | 'contribution_amount', 42 | 'ip', 43 | 'kyc_result', 44 | 'idm_result', 45 | 'decline_reason', 46 | 'onfid_status', 47 | 'info', 48 | 'action', 49 | 'eth_cap', 50 | ] 51 | created_at = DateTimeField(default=datetime.datetime.utcnow) 52 | 53 | state = StringField(choices=ALL_STATES, default=NEW_USER) 54 | decline_reason = StringField(choices=ALL_DECLINE_REASONS) 55 | info = StringField() 56 | 57 | ip_country = StringField() 58 | dob = DateTimeField() 59 | email = EmailField(unique=True, required=True) 60 | eth_address = StringField(unique=True, sparse=True) 61 | eth_amount = FloatField() 62 | eth_cap = FloatField() 63 | first_name = StringField() 64 | last_name = StringField() 65 | phone = StringField() 66 | ip = StringField() 67 | dfp = StringField() 68 | telegram = StringField(unique=True) 69 | confirmed_location = BooleanField() 70 | idm_tid = IntField() 71 | 72 | onfid_id = StringField() 73 | onfid_status = StringField() 74 | 75 | address = StringField() 76 | city = StringField() 77 | state_code = StringField() 78 | zip_code = StringField() 79 | country_code = StringField() 80 | 81 | kyc_result = StringField() 82 | idm_result = StringField() 83 | doc_type = StringField() 84 | 85 | contribution_amount = FloatField(default=0) 86 | contribution_tx = ListField(StringField()) 87 | 88 | medium = StringField() 89 | reddit = StringField() 90 | twitter = StringField() 91 | linkedin = StringField() 92 | facebook = StringField() 93 | 94 | on_create = Signal() 95 | on_transition = Signal() 96 | 97 | @classmethod 98 | def find(cls, uid: ObjectId) -> 'User': 99 | """ Helper to find user in DB or raise an error. """ 100 | user = cls.objects(id=uid).first() 101 | if not user: 102 | raise UserNotFound() 103 | 104 | return user 105 | 106 | @classmethod 107 | def create(cls, **data) -> 'User': 108 | obj = cls(id=ObjectId(), **data).save(force_insert=True) 109 | cls.on_create.send(obj) 110 | return obj 111 | 112 | def to_json(self): 113 | return { 114 | 'address': self.address, 115 | 'city': self.city, 116 | 'state_code': self.state_code, 117 | 'zip_code': self.zip_code, 118 | 'country_code': self.country_code, 119 | 'dob': int(self.dob.strftime('%s')) if self.dob else None, 120 | 'email': self.email, 121 | # 'telegram': self.telegram, 122 | 'confirmed_location': self.confirmed_location, 123 | 'eth_address': self.eth_address, 124 | 'eth_amount': self.eth_amount, 125 | 'state': self.state, 126 | 'eth_cap': self.eth_cap, 127 | 'decline_reason': self.decline_reason, 128 | 'id': str(self.id), 129 | 'first_name': self.first_name, 130 | 'last_name': self.last_name, 131 | 'phone': self.phone, 132 | 'token': get_token(self.id), 133 | } 134 | 135 | def to_csv(self) -> dict: 136 | res = {} 137 | for field in self.CSV_FIELDS: 138 | if field == 'action': 139 | value = '' 140 | elif field == 'email': 141 | name, domain = self.email.split('@') 142 | value = f'{name[0]}...{name[-1]}@{domain}' 143 | elif field == 'id': 144 | value = str(self.id) 145 | elif field in ['created_at', 'dob']: 146 | value = self[field].isoformat() 147 | elif field == 'telegram': 148 | value = f'{self.telegram[0]}...{self.telegram[-1]}' if self.telegram else None 149 | else: 150 | value = self[field] 151 | res.update({field: value}) 152 | return res 153 | 154 | def transition(self, new_state: str, decline_reason: str = None, details: str = None) -> str: 155 | """ 156 | Transition user to new state. 157 | 158 | This will trigger signal to perform operations required for new state. 159 | 160 | :return: Updated state after all signals 161 | 162 | """ 163 | previous = self.state 164 | self.state = new_state 165 | if decline_reason: 166 | self.decline_reason = decline_reason 167 | 168 | if details: 169 | self.info = details 170 | 171 | self.save() 172 | self.on_transition.send(self, transition=(previous, self.state)) 173 | 174 | # Our state can be changed after signals are completed 175 | self.reload() 176 | return self.state 177 | 178 | def __str__(self): 179 | return f'' 180 | -------------------------------------------------------------------------------- /app/user/state.py: -------------------------------------------------------------------------------- 1 | # User states 2 | # --------------------------- 3 | NEW_USER = 'new_user' #: User didn't perform any action. He's not in our DB yet. 4 | INFO_NOT_VERIFIED = 'info_not_verified' #: Basic info provided, not yet verified by IDM 5 | INFO_PENDING_VERIFICATION = 'info_pending_verification' #: Info provided, submitted to IDM, waiting for response 6 | INFO_VERIFIED = 'info_verified' #: Basic info was verified by IDM 7 | INFO_DECLINED = 'info_declined' #: Basic info was declined by IDM 8 | INFO_FAILED = 'info_failed' #: Failed to make a request to the IDM 9 | 10 | ID_NOT_VERIFIED = 'id_not_verified' #: ID pic provided but not submitted to IDM yet 11 | ID_PENDING_VERIFICATION = 'id_pending_verification' #: ID submitted to IDM, waiting for verification 12 | ID_VERIFIED = 'id_verified' #: IDM verified ID 13 | ID_DECLINED = 'id_declined' #: IDM declined ID pic 14 | ID_FAILED = 'id_failed' #: Failed to make id verification request to the IDM 15 | 16 | CONTRIBUTED = 'contributed' #: User successfully contributed to our ICO. Transaction has landed in the blockchain 17 | APPROVED_NO_CAP = 'approved_no_cap' #: Admin approved user, no cap set yet 18 | APPROVED_CAP = 'approved_cap' #: Admin approved and set personal token cap 19 | DECLINED = 'declined' #: Admin declined the user 20 | 21 | STATE_FLOW = [ 22 | NEW_USER, 23 | INFO_NOT_VERIFIED, 24 | INFO_PENDING_VERIFICATION, 25 | INFO_VERIFIED, 26 | ID_NOT_VERIFIED, 27 | ID_PENDING_VERIFICATION, 28 | ID_VERIFIED, 29 | APPROVED_NO_CAP, 30 | APPROVED_CAP, 31 | CONTRIBUTED, 32 | ] 33 | 34 | FINAL_STATES = [ 35 | DECLINED, 36 | ID_DECLINED, 37 | ID_FAILED, 38 | INFO_DECLINED, 39 | INFO_FAILED, 40 | ] 41 | 42 | ALL_STATES = STATE_FLOW + FINAL_STATES 43 | # --------------------------- 44 | 45 | BANNED_COUNTRIES = ['CN', 'US', 'TW', 'HK'] 46 | 47 | # Decline reasons 48 | DECLINE_COUNTRY = 'decline_country' #: Provided country is blacklisted 49 | DECLINE_IDM_INFO = 'decline_idm_info' #: Declined by the IDM for basic info 50 | DECLINE_IDM_ID = 'decline_idm_id' #: Declined by the IDM for ID pic 51 | DECLINE_ADMIN = 'decline_admin' #: Admin declined 52 | 53 | ALL_DECLINE_REASONS = [ 54 | DECLINE_COUNTRY, 55 | DECLINE_IDM_INFO, 56 | DECLINE_IDM_ID, 57 | DECLINE_ADMIN, 58 | ] 59 | -------------------------------------------------------------------------------- /app/user/test_auth.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from mock import patch 4 | 5 | from conftest import error 6 | from errors import WhitelistClosed 7 | from tokens import get_token 8 | from user.test_user_info import new_user 9 | 10 | 11 | def test_valid_token_auth(service, user): 12 | res = service.get('/v1/user', auth=get_token(user.id)) 13 | assert res.status_code == 200 14 | 15 | 16 | def test_closed_whitelist(service): 17 | with patch('user.views.WHITELIST_CLOSED', True): 18 | res = service.post('/v1/user', new_user()) 19 | assert error(res, WhitelistClosed) 20 | 21 | 22 | def test_whitelist_not_yet_opened(service): 23 | ts = datetime.datetime(2020, 10, 10) 24 | with patch('user.views.WHITELIST_OPEN_DATE', ts): 25 | res = service.post('/v1/user', new_user()) 26 | assert error(res, WhitelistClosed) 27 | -------------------------------------------------------------------------------- /app/user/test_user_info.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import pytest 4 | from mock import patch 5 | from requests import HTTPError, Response 6 | 7 | from conftest import error 8 | from errors import ObjectExists, ValidationError 9 | from idm.const import STATUS_ACCEPTED, STATUS_DECLINED, USER_REPUTATION_SUSPICIOUS 10 | from idm.errors import IDMError 11 | from idm.models import IDMResponse 12 | from tokens import get_token 13 | from user.models import User 14 | from user.state import DECLINE_COUNTRY, INFO_DECLINED, INFO_FAILED, INFO_PENDING_VERIFICATION, INFO_VERIFIED 15 | from user.views import to_eth 16 | 17 | 18 | def new_user(**kwargs) -> dict: 19 | default = { 20 | 'email': kwargs.get('email', 'user@example.com'), 21 | 'dob': int(datetime.datetime(2010, 10, 10).strftime('%s')), 22 | 'dfp': '{"some":"data", "is": "here"}', 23 | 'phone': '+1234567890', 24 | 'eth_address': kwargs.get('eth_address', '0x29D7d1dd5B6f9C864d9db560D72a247c178aE86B'), 25 | 'eth_amount': 1.23, 26 | 'address': 'blappy blop', 27 | 'city': 'blappy blop', 28 | 'state_code': 'CA', 29 | 'zip_code': '12345', 30 | 'country_code': 'AU', 31 | 'telegram': '@telegram', 32 | 'confirmed_location': True, 33 | 'first_name': 'Homer', 34 | 'last_name': 'Simpson', 35 | } 36 | if 'dob' in kwargs: 37 | kwargs['dob'] = kwargs['dob'].strftime('%s') 38 | default.update(kwargs) 39 | return default 40 | 41 | 42 | def test_to_eth(): 43 | failed = [ 44 | 'DEADBEEF', 45 | '0x12345', 46 | '0x12345', 47 | ] 48 | for item in failed: 49 | with pytest.raises(ValueError): 50 | to_eth(item) 51 | 52 | assert to_eth('0x29D7d1dd5B6f9C864d9db560D72a247c178aE86B') == '29D7d1dd5B6f9C864d9db560D72a247c178aE86B' 53 | assert to_eth('29D7d1dd5B6f9C864d9db560D72a247c178aE86B') == '29D7d1dd5B6f9C864d9db560D72a247c178aE86B' 54 | 55 | 56 | def test_new_user(service): 57 | data = new_user() 58 | res = service.post('/v1/user', data) 59 | assert res.status_code == 201, res.json 60 | 61 | user = User.objects.get() 62 | assert user.email == data['email'] 63 | assert user.eth_address == '29D7d1dd5B6f9C864d9db560D72a247c178aE86B' 64 | 65 | 66 | def test_get_updated_info(service): 67 | data = new_user() 68 | res = service.post('/v1/user', data) 69 | assert res.status_code == 201, res.json 70 | assert res.json['email'] == data['email'] 71 | assert res.json['dob'] == data['dob'] 72 | 73 | 74 | def test_add_dob(service): 75 | dt = datetime.datetime(2010, 10, 10) 76 | 77 | res = service.post('/v1/user', new_user(dob=dt)) 78 | assert res.status_code == 201, res.json 79 | 80 | user = User.objects.get() 81 | assert user.dob == dt 82 | 83 | 84 | def test_interrupted_verification(service): 85 | """ Make sure user remains in INFO_PENDING if IDM verification fails. """ 86 | # Raise exception during IDM verification 87 | with patch('user.verifications.verify', side_effect=Exception): 88 | with pytest.raises(Exception): 89 | service.post('/v1/user', new_user()) 90 | 91 | user = User.objects.get() 92 | assert user.state == INFO_PENDING_VERIFICATION 93 | 94 | 95 | def test_failed_verification_request(service): 96 | with patch('user.verifications.verify', side_effect=IDMError(HTTPError('blop', response=Response()))): 97 | service.post('/v1/user', new_user()) 98 | 99 | user = User.objects.get() 100 | assert user.state == INFO_FAILED 101 | # assert user.info == 'blop' 102 | 103 | 104 | def test_successful_verification(service): 105 | with patch('user.verifications.verify', return_value=IDMResponse(result=STATUS_ACCEPTED, transaction_id='123')): 106 | service.post('/v1/user', new_user()) 107 | 108 | user = User.objects.get() 109 | assert user.state == INFO_VERIFIED 110 | 111 | 112 | def test_declined_verification(service): 113 | response = IDMResponse(result=STATUS_DECLINED, transaction_id='123', user_reputation=USER_REPUTATION_SUSPICIOUS) 114 | 115 | with patch('user.verifications.verify', return_value=response): 116 | service.post('/v1/user', new_user()) 117 | 118 | user = User.objects.get() 119 | assert user.state == INFO_DECLINED 120 | assert user.info == USER_REPUTATION_SUSPICIOUS 121 | 122 | 123 | def test_declined_by_pending_verification(service): 124 | """ Make sure we stop handling IDM if user was declined within INFO_PENDING_VERIFICATION state. """ 125 | 126 | def decline_user(user: User, transition): 127 | """ Decline user as it reaches INFO_PENDING_VERIFICATION state. """ 128 | if transition[1] == INFO_PENDING_VERIFICATION: 129 | user.transition(INFO_DECLINED) 130 | 131 | with User.on_transition.connected_to(decline_user): 132 | with patch('idm.verify.verify') as idm_call: 133 | service.post('/v1/user', new_user()) 134 | idm_call.assert_not_called() 135 | 136 | user = User.objects.get() 137 | assert user.state == INFO_DECLINED 138 | 139 | 140 | def test_user_token(service): 141 | res = service.post('/v1/user', new_user()) 142 | assert res.status_code == 201, res.json 143 | 144 | user = User.objects.get() 145 | assert res.json['token'] == get_token(user.id) 146 | 147 | 148 | def test_start_as_existing_user(service, user): 149 | res = service.post('/v1/user', new_user(email=user.email, eth_address=user.eth_address)) 150 | assert res.status_code == 200, res.json 151 | 152 | # This will be the same user 153 | assert res.json['id'] == str(user.id) 154 | assert User.objects.count() == 1 155 | 156 | 157 | def test_try_using_existing_eth(service, user): 158 | res = service.post('/v1/user', new_user(email='another@example.com', eth_address=user.eth_address)) 159 | assert error(res, ObjectExists) 160 | 161 | 162 | def test_try_using_existing_email(service, user): 163 | res = service.post('/v1/user', new_user(email=user.email, eth_address='0x1111111111111111111111111111111111111111')) 164 | assert error(res, ObjectExists) 165 | 166 | 167 | @pytest.mark.skip 168 | def test_banned_country(service): 169 | with patch('user.verifications.BANNED_COUNTRIES', ['AU']): 170 | res = service.post('/v1/user', new_user(), **{'CF-IPCountry': 'AU'}) 171 | assert res.status_code == 201 172 | assert res.json['state'] == INFO_DECLINED 173 | assert res.json['decline_reason'] == DECLINE_COUNTRY 174 | 175 | 176 | def test_not_confirmed_location(service): 177 | res = service.post('/v1/user', new_user(confirmed_location=False)) 178 | assert error(res, ValidationError) 179 | 180 | 181 | def test_invalid_telegram(service): 182 | res = service.post('/v1/user', new_user(telegram='no')) 183 | assert error(res, ValidationError) 184 | 185 | 186 | def test_two_telegrams(service, user): 187 | res = service.post('/v1/user', new_user(telegram='telegram')) 188 | assert error(res, ObjectExists) 189 | 190 | 191 | def test_missing_eth(service): 192 | data = new_user() 193 | data.pop('eth_address') 194 | res = service.post('/v1/user', data) 195 | assert res.status_code == 201, res.json 196 | -------------------------------------------------------------------------------- /app/user/verifications.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from idm.errors import IDMError 4 | from idm.models import IDMResponse 5 | from idm.const import STATUS_ACCEPTED, STATUS_DECLINED, STATUS_PENDING 6 | from idm.verify import verify 7 | from ids.models import IDUpload 8 | from user.models import User 9 | from user.state import ( 10 | DECLINE_IDM_INFO, 11 | ID_FAILED, 12 | ID_PENDING_VERIFICATION, 13 | ID_VERIFIED, 14 | INFO_DECLINED, 15 | INFO_FAILED, 16 | INFO_PENDING_VERIFICATION, 17 | INFO_VERIFIED, 18 | ) 19 | 20 | log = logging.getLogger(__name__) 21 | 22 | 23 | def verify_info(user: User) -> str: 24 | state = user.transition(INFO_PENDING_VERIFICATION) 25 | 26 | # Make sure that state didn't changed after we reached pending verification state 27 | if state != INFO_PENDING_VERIFICATION: 28 | return state 29 | 30 | try: 31 | response = verify(user, kyc=True) 32 | apply_response(user, response, True) 33 | except IDMError as err: 34 | return user.transition(INFO_FAILED, details=err.text) 35 | 36 | 37 | def verify_ids(upload: IDUpload) -> str: 38 | user = upload.user 39 | state = user.transition(ID_PENDING_VERIFICATION) 40 | 41 | # Make sure that state didn't changed after we reached pending verification state 42 | if state != ID_PENDING_VERIFICATION: 43 | return state 44 | 45 | image1 = upload.upload1.to_base64() 46 | image2 = upload.upload2.to_base64() 47 | 48 | try: 49 | response = verify( 50 | user=user, 51 | kyc=False, 52 | image1=image1, 53 | image2=image2, 54 | doc_type=upload.doc_type, 55 | doc_state=upload.doc_state, 56 | doc_country=upload.doc_country, 57 | ) 58 | if response.user: 59 | apply_response(user, response, False) 60 | except IDMError as err: 61 | return user.transition(ID_FAILED, details=err.text) 62 | 63 | 64 | def apply_response(user: User, response: IDMResponse, is_info: bool) -> str: 65 | if is_info: 66 | user.update(kyc_result=response.status) 67 | else: 68 | user.update(idm_result=response.status) 69 | 70 | if response.status == STATUS_DECLINED: 71 | if is_info: # KYC is hard-fail 72 | return user.transition( 73 | INFO_DECLINED, 74 | decline_reason=DECLINE_IDM_INFO, 75 | details=response.user_reputation, 76 | ) 77 | else: # IDs are soft-fail - we do nothing and let admin decide 78 | return user.state 79 | elif response.status in [STATUS_ACCEPTED, STATUS_PENDING]: 80 | return user.transition(INFO_VERIFIED if is_info else ID_VERIFIED) 81 | else: 82 | log.warning('Got different response from IDM: %s', response.status) 83 | return user.state 84 | -------------------------------------------------------------------------------- /app/user/views.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, jsonify, request 2 | from voluptuous import All, Any, Coerce, Email, Length, Optional, REMOVE_EXTRA, Range, datetime, re 3 | from werkzeug.exceptions import NotFound 4 | 5 | from config import ETH_ADDRESS, WHITELISTED_ADDRESSES, WHITELIST_CLOSED, WHITELIST_OPEN_DATE, ETH_MAX_CONTRIBUTION 6 | from errors import ValidationError, WhitelistClosed 7 | from schema import Schema 8 | from user.auth import authenticate 9 | from user.models import User 10 | from user.state import INFO_NOT_VERIFIED 11 | from user.verifications import verify_info 12 | 13 | blueprint = Blueprint('info', __name__) 14 | 15 | 16 | def to_eth(value: str) -> str: 17 | match = re.match('^(0x)?(?P[0-9a-f]{40})$', value, flags=re.IGNORECASE) 18 | if not match: 19 | raise ValueError(value) 20 | return match.group('addr').lower() 21 | 22 | 23 | def to_datetime(value: str) -> datetime.datetime: 24 | try: 25 | return datetime.datetime.utcfromtimestamp(int(value)) 26 | except: 27 | raise ValueError(value) 28 | 29 | 30 | def to_telegram(value: str) -> str: 31 | match = re.match('^@?(?P[0-9a-z_]{5,25})$', value, flags=re.IGNORECASE) 32 | if not match: 33 | raise ValueError(value) 34 | return match.group('name') 35 | 36 | 37 | @blueprint.route('/v1/user', methods=['POST']) 38 | def provide_info(): 39 | if WHITELIST_CLOSED: 40 | raise WhitelistClosed() 41 | 42 | if WHITELIST_OPEN_DATE and WHITELIST_OPEN_DATE > datetime.datetime.utcnow(): 43 | raise WhitelistClosed(details=dict(open_ts=int(WHITELIST_OPEN_DATE.strftime('%s')))) 44 | 45 | schema = Schema({ 46 | 'first_name': All(Length(2, 30), str), 47 | 'last_name': All(Length(2, 30), str), 48 | 'email': Email(), 49 | 'dob': Coerce(to_datetime), 50 | 'address': All(Length(1, 100), str), 51 | 'city': All(Length(2, 30), str), 52 | Optional('state_code', default=None): Any(None, All(Length(0, 30), str)), 53 | 'zip_code': All(Length(2, 20), str), 54 | 'country_code': All(Length(2, 3), str), 55 | 'phone': All(Length(8, 20), str), 56 | Optional('eth_address', default=None): Coerce(to_eth), 57 | Optional('eth_amount', default=None): All(Range(0, 100), Any(float, int)), 58 | 'telegram': Coerce(to_telegram), 59 | 'confirmed_location': bool, 60 | 'dfp': All(Length(10, 4096), str), 61 | Optional('medium', default=None): Any(None, All(Length(0, 150), str)), 62 | Optional('reddit', default=None): Any(None, All(Length(0, 150), str)), 63 | Optional('twitter', default=None): Any(None, All(Length(0, 150), str)), 64 | Optional('linkedin', default=None): Any(None, All(Length(0, 150), str)), 65 | Optional('facebook', default=None): Any(None, All(Length(0, 150), str)), 66 | }, extra=REMOVE_EXTRA, required=True) 67 | data = schema(request.json) 68 | data['email'] = data['email'].lower() 69 | 70 | if not data['confirmed_location']: 71 | raise ValidationError('Unconfirmed location') 72 | 73 | data['ip'] = request.remote_addr 74 | data['ip_country'] = request.headers.get('CF-IPCountry') 75 | 76 | # Try to find existing user 77 | user = User.objects(eth_address=data['eth_address']).first() 78 | if user: 79 | return jsonify(user.to_json()) 80 | 81 | ####################### WHITELIST IS CLOSED FOR NEW USERS #################### 82 | raise WhitelistClosed() 83 | 84 | user = User.create(**data) 85 | 86 | status = user.transition(INFO_NOT_VERIFIED) 87 | 88 | if status == INFO_NOT_VERIFIED: 89 | verify_info(user) 90 | 91 | return jsonify(user.reload().to_json()), 201 92 | 93 | 94 | @blueprint.route('/v1/tokensale/', methods=['GET']) 95 | def address_is_whitelisted(addr: str): 96 | if datetime.datetime.utcnow() < datetime.datetime(2018, 2, 21, 7, 0, 0, 0, tzinfo=None): 97 | raise NotFound() 98 | try: 99 | addr = to_eth(addr) 100 | except ValueError: 101 | raise NotFound() 102 | 103 | if addr in WHITELISTED_ADDRESSES: 104 | return jsonify({"address": ETH_ADDRESS, 'max_contribution': ETH_MAX_CONTRIBUTION}) 105 | else: 106 | raise NotFound() 107 | 108 | 109 | @blueprint.route('/v1/user', methods=['GET']) 110 | @authenticate(bypass_closing=True) 111 | def get_public_info(user: User): 112 | return jsonify(user.to_json()) 113 | 114 | 115 | @blueprint.route('/v1/user/', methods=['GET']) 116 | def get_info_by_eth(addr: str): 117 | try: 118 | addr = to_eth(addr) 119 | except ValueError: 120 | return jsonify({'status': 'not-found'}) 121 | 122 | user = User.objects(eth_address=addr).first() 123 | if not user: 124 | return jsonify({'status': 'not-found'}) 125 | 126 | status = { 127 | 'clear': 'approved', 128 | 'consider': 'declined' 129 | } 130 | return jsonify({ 131 | 'status': status.get(user.onfid_status, 'declined'), 132 | }) 133 | 134 | 135 | @blueprint.route('/v1/status', methods=['GET', 'POST']) 136 | def whitelist_status(): 137 | needs_closed = request.args.get('close') 138 | if needs_closed or WHITELIST_CLOSED or (WHITELIST_OPEN_DATE and WHITELIST_OPEN_DATE > datetime.datetime.utcnow()): 139 | raise WhitelistClosed() 140 | 141 | return jsonify({'status': 'open'}) 142 | -------------------------------------------------------------------------------- /app/uwsgi.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | http-socket = :8080 3 | socket = :8888 4 | callable = app 5 | chdir = /app/ 6 | wsgi-file = wsgi.py 7 | need-app = true 8 | gevent = 1024 9 | listen = 512 10 | gevent-monkey-patch = true 11 | -------------------------------------------------------------------------------- /app/wsgi.py: -------------------------------------------------------------------------------- 1 | from gevent import monkey 2 | monkey.patch_all() 3 | 4 | import logging 5 | import os 6 | 7 | from app import create_app 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | app = create_app(os.environ.get('CONFIG', 'prod')) 12 | 13 | if __name__ == '__main__': 14 | log.info('Serving requests') 15 | app.run(host='0.0.0.0', port=8080) 16 | --------------------------------------------------------------------------------