├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── manage.py ├── pypi_portal ├── __init__.py ├── application.py ├── blueprints.py ├── config.py ├── core │ ├── __init__.py │ ├── email.py │ └── flash.py ├── extensions.py ├── middleware.py ├── models │ ├── __init__.py │ ├── helpers.py │ ├── pypi.py │ └── redis.py ├── static │ ├── favicon.ico │ └── logo.svg ├── tasks │ ├── __init__.py │ └── pypi.py ├── templates │ ├── 400.html │ ├── 403.html │ ├── 404.html │ ├── 500.html │ ├── base.html │ ├── email.html │ ├── flash.html │ └── navbar.html └── views │ ├── __init__.py │ ├── examples │ ├── __init__.py │ ├── alerts.py │ ├── exception.py │ └── templates │ │ └── examples_alerts.html │ ├── home │ ├── __init__.py │ ├── index.py │ └── templates │ │ └── home_index.html │ └── pypi │ ├── __init__.py │ ├── packages.py │ └── templates │ ├── pypi_packages.html │ └── pypi_packages_paginator.html ├── requirements.txt └── tests ├── __init__.py ├── conftest.py ├── core └── test_email.py ├── models ├── __init__.py └── test_helpers.py ├── tasks ├── conftest.py └── test_pypi.py ├── test_application.py ├── test_basics.py ├── test_blueprints.py ├── test_middleware.py └── views ├── __init__.py ├── test_examples_alerts.py ├── test_examples_exception.py └── test_pypi_packages.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | bin/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # Installer logs 26 | pip-log.txt 27 | pip-delete-this-directory.txt 28 | 29 | # Unit test / coverage reports 30 | htmlcov/ 31 | .tox/ 32 | .coverage 33 | .cache 34 | nosetests.xml 35 | coverage.xml 36 | 37 | # Translations 38 | *.mo 39 | 40 | # Mr Developer 41 | .mr.developer.cfg 42 | .project 43 | .pydevproject 44 | 45 | # Rope 46 | .ropeproject 47 | 48 | # Django stuff: 49 | *.log 50 | *.pot 51 | 52 | # Sphinx documentation 53 | docs/_build/ 54 | 55 | # Project specific 56 | pypi_portal/*/flask_statics_helper 57 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - 2.7 5 | 6 | services: 7 | - redis-server 8 | - mysql 9 | 10 | install: pip install -r requirements.txt pytest-cov python-coveralls 11 | 12 | before_script: 13 | - mysql -e "CREATE DATABASE pypi_portal_testing;" 14 | - 'echo -e "_SQLALCHEMY_DATABASE_USERNAME: travis\n_SQLALCHEMY_DATABASE_PASSWORD: \"\"" > config.yml' 15 | 16 | script: make test 17 | 18 | after_success: coveralls -i 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Robpol86 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYCODE = import flask.ext.statics as a; print a.__path__[0] 2 | .PHONY: default isvirtualenv 3 | 4 | default: 5 | @echo "Local examples:" 6 | @echo " make run # Starts a Flask development server locally." 7 | @echo " make shell # Runs 'manage.py shell' locally with iPython." 8 | @echo " make celery # Runs one development Celery worker with Beat." 9 | @echo " make style # Check code styling with flake8." 10 | @echo " make lint # Runs PyLint." 11 | @echo " make test # Tests entire application with pytest." 12 | 13 | isvirtualenv: 14 | @if [ -z "$(VIRTUAL_ENV)" ]; then echo "ERROR: Not in a virtualenv." 1>&2; exit 1; fi 15 | 16 | run: 17 | (((i=0; while \[ $$i -lt 40 \]; do sleep 0.5; i=$$((i+1)); \ 18 | netstat -anp tcp |grep "\.5000.*LISTEN" &>/dev/null && break; done) && open http://localhost:5000/) &) 19 | ./manage.py devserver 20 | 21 | shell: 22 | ./manage.py shell 23 | 24 | celery: 25 | ./manage.py celerydev 26 | 27 | style: 28 | flake8 --max-line-length=120 --statistics pypi_portal 29 | 30 | lint: 31 | pylint --max-line-length=120 pypi_portal 32 | 33 | test: 34 | py.test --cov-report term-missing --cov pypi_portal tests 35 | 36 | testpdb: 37 | py.test --pdb tests 38 | 39 | testcovweb: 40 | py.test --cov-report html --cov pypi_portal tests 41 | open htmlcov/index.html 42 | 43 | pipinstall: isvirtualenv 44 | # For development environments. Symlinks are for PyCharm inspections to work with Flask-Statics-Helper. 45 | pip install -r requirements.txt flake8 pylint ipython pytest-cov 46 | [ -h pypi_portal/templates/flask_statics_helper ] || ln -s `python -c "$(PYCODE)"`/templates/flask_statics_helper \ 47 | pypi_portal/templates/flask_statics_helper 48 | [ -h pypi_portal/static/flask_statics_helper ] || ln -s `python -c "$(PYCODE)"`/static \ 49 | pypi_portal/static/flask_statics_helper 50 | 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flask-Large-Application-Example 2 | 3 | PyPI Portal is a small demo app used as an example of a potentially large Flask application with several views and 4 | Celery tasks. This is how I structure my large Flask applications. In this README I'll explain my design choices with 5 | several aspects of the project. 6 | 7 | For information on how to deploy this application to different production environments, visit 8 | [the project's wiki](https://github.com/Robpol86/Flask-Large-Application-Example/wiki). 9 | 10 | [![Build Status](https://travis-ci.org/Robpol86/Flask-Large-Application-Example.svg?branch=master)] 11 | (https://travis-ci.org/Robpol86/Flask-Large-Application-Example) 12 | [![Coverage Status](https://img.shields.io/coveralls/Robpol86/Flask-Large-Application-Example.svg)] 13 | (https://coveralls.io/r/Robpol86/Flask-Large-Application-Example) 14 | 15 | ## Features 16 | 17 | Some features I've included in this demo application are: 18 | 19 | * Tests are written for [pytest](http://pytest.org/). 20 | * Tasks/scripts/jobs are supported with [Celery](http://www.celeryproject.org/). 21 | * Any unhandled exceptions raised in views or Celery tasks are emailed to you from your production instance. The email 22 | is styled to look similar to the exceptions shown in development environments, but without the interactive console. 23 | * Message flashing is "powered by" [Bootstrap Growl](https://github.com/mouse0270/bootstrap-growl) and I've also 24 | included Bootstrap Modals and Wells as flashed message containers. More about that in `core/flash.py`. 25 | 26 | ## Directory Structure 27 | 28 | ```GAP 29 | ├─ pypi_portal/ # All application code in this directory. 30 | │ ├─ core/ # Shared/misc code goes in here as packages or modules. 31 | │ ├─ models/ 32 | │ │ ├─ fruit.py # Holds several tables about a subject. 33 | │ │ └─ vegetable.py 34 | │ │ 35 | │ ├─ static/ 36 | │ │ ├─ favicon.ico 37 | │ │ └─ some_lib/ 38 | │ │ ├─ css/ 39 | │ │ │ └─ some_lib.css 40 | │ │ └─ js/ 41 | │ │ └─ some_lib.js 42 | │ │ 43 | │ ├─ tasks/ # Celery tasks (packages or modules). 44 | │ ├─ templates/ # Base templates used/included throughout the app. 45 | │ │ ├─ 404.html 46 | │ │ └─ base.html 47 | │ │ 48 | │ ├─ views/ 49 | │ │ ├─ view1/ 50 | │ │ │ ├─ templates/ # Templates only used by view1. 51 | │ │ │ │ └─ view1_section1.html # Naming convention: package_module.html 52 | │ │ │ ├─ section1.py # Each view module has its own blueprint. 53 | │ │ │ └─ section2.py 54 | │ │ │ 55 | │ │ ├─ view2/ 56 | │ │ └─ view3/ 57 | │ │ 58 | │ ├─ application.py # Flask create_app() factory. 59 | │ ├─ blueprints.py # Define Flask blueprints and their URLs. 60 | │ ├─ config.py # All configs for Flask, Celery, Prod, Dev, etc. 61 | │ ├─ extensions.py # Instantiate SQLAlchemy, Celery, etc. Importable. 62 | │ └─ middleware.py # Error handlers, template filters, other misc code. 63 | │ 64 | ├─ tests/ # Tests are structured similar to the application. 65 | │ ├─ core/ 66 | │ │ └─ test_something.py 67 | │ ├─ tasks/ 68 | │ └─ conftest.py 69 | │ 70 | └─ manage.py # Main entry-point into the Flask/Celery application. 71 | ``` 72 | 73 | ## Design Choices 74 | 75 | ### Blueprints 76 | 77 | The first thing you may notice are where blueprints are defined. Flask applications usually define their blueprints 78 | inside view modules themselves, and must be imported in or after `create_app()`. URLs for blueprints are usually set in 79 | or after `create_app()` as well. 80 | 81 | I've never liked defining blueprints in the views since according to pep8 the variables should be IN_ALL_CAPS (it's true 82 | that blueprints are still module-level in `blueprints.py` but since that file is 99% module-level variables I make a 83 | small exception to pep8 and keep it lower case), plus usually it's the only module-level variable in the file. 84 | 85 | Instead I define blueprints in `blueprints.py` and import them in both views and `application.py`. While I'm at it, I 86 | "centralize" URL and module specifications in blueprints.py instead of having those two pieces of information in views 87 | and `application.py`. 88 | 89 | ### Templates 90 | 91 | This is another deviation from usual Flask applications. Instead of dumping all templates used by all views into one 92 | directory, I split it up into two classes: "common" and "per-view". This makes it way easier to determine which view a 93 | template is used in with a quick glance. In very large applications this is much more manageable, since having tens or 94 | even hundreds of templates in one directory is ugly. 95 | 96 | First I have my common templates, located in the usual `templates` directory at the base of the application directory 97 | structure. Templates not tied to a specific view go here. 98 | 99 | Then I have the per-view templates. Each view package will have its own `templates` directory. There is one problem 100 | though: Flask flattens the template's directory structure into one directory. `templates/base.html` and 101 | `views/my_view/templates/base.html` will collide. To get around this, templates inside a per-view template directory are 102 | formatted as packageName_moduleName.html (where packageName is my_view). When modules have a lot of templates just 103 | append to the filename (e.g. packageName_moduleName_feature.html). 104 | 105 | ### Extensions 106 | 107 | This isn't very unique but I'll cover it anyway. I've seen other projects follow this convention. The idea is to 108 | instantiate extensions such as SQLAlchemy here, but without any arguments (and without calling `init_app()`). 109 | 110 | These may be imported by views/tasks and are also imported by `application.py` which is where `init_app()` is called. 111 | 112 | ### Config 113 | 114 | I elected to keep all configurations in this one file, instead of having different files for different environments 115 | (prod, stage, dev, etc). One important note is I also keep non-Flask configurations in the same file (e.g. Celery, 116 | SQLAlchemy, even hard-coded values). If you need to hard-code some data that's shared among different modules, I'd put 117 | it in config.py. If you need to hard-code data that's only used in one module (just one view for example), then I'd keep 118 | it in that module as a module-level variable. 119 | 120 | I structure my `config.py` with several classes, inheriting from the previous one to avoid duplicating data. 121 | 122 | ### Tests 123 | 124 | The tests directory structure mirrors the application's. This makes it easy to group tests for specific views/modules. 125 | If a module such as `core/email.py` requires several tests, I would split them up into different test modules inside a 126 | package such as `tests/core/email/test_feature1.py` and so on. 127 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | """Main entry-point into the 'PyPI Portal' Flask and Celery application. 3 | 4 | This is a demo Flask application used to show how I structure my large Flask 5 | applications. 6 | 7 | License: MIT 8 | Website: https://github.com/Robpol86/Flask-Large-Application-Example 9 | 10 | Command details: 11 | devserver Run the application using the Flask Development 12 | Server. Auto-reloads files when they change. 13 | tornadoserver Run the application with Facebook's Tornado web 14 | server. Forks into multiple processes to handle 15 | several requests. 16 | celerydev Starts a Celery worker with Celery Beat in the same 17 | process. 18 | celerybeat Run a Celery Beat periodic task scheduler. 19 | celeryworker Run a Celery worker process. 20 | shell Starts a Python interactive shell with the Flask 21 | application context. 22 | create_all Only create database tables if they don't exist and 23 | then exit. 24 | 25 | Usage: 26 | manage.py devserver [-p NUM] [-l DIR] [--config_prod] 27 | manage.py tornadoserver [-p NUM] [-l DIR] [--config_prod] 28 | manage.py celerydev [-l DIR] [--config_prod] 29 | manage.py celerybeat [-s FILE] [--pid=FILE] [-l DIR] [--config_prod] 30 | manage.py celeryworker [-n NUM] [-l DIR] [--config_prod] 31 | manage.py shell [--config_prod] 32 | manage.py create_all [--config_prod] 33 | manage.py (-h | --help) 34 | 35 | Options: 36 | --config_prod Load the production configuration instead of 37 | development. 38 | -l DIR --log_dir=DIR Log all statements to file in this directory 39 | instead of stdout. 40 | Only ERROR statements will go to stdout. stderr 41 | is not used. 42 | -n NUM --name=NUM Celery Worker name integer. 43 | [default: 1] 44 | --pid=FILE Celery Beat PID file. 45 | [default: ./celery_beat.pid] 46 | -p NUM --port=NUM Flask will listen on this port number. 47 | [default: 5000] 48 | -s FILE --schedule=FILE Celery Beat schedule database file. 49 | [default: ./celery_beat.db] 50 | """ 51 | 52 | from __future__ import print_function 53 | from functools import wraps 54 | import logging 55 | import logging.handlers 56 | import os 57 | import signal 58 | import sys 59 | 60 | from celery.app.log import Logging 61 | from celery.bin.celery import main as celery_main 62 | from docopt import docopt 63 | import flask 64 | from flask.ext.script import Shell 65 | from tornado import httpserver, ioloop, web, wsgi 66 | 67 | from pypi_portal.application import create_app, get_config 68 | from pypi_portal.extensions import db 69 | 70 | OPTIONS = docopt(__doc__) if __name__ == '__main__' else dict() 71 | 72 | 73 | class CustomFormatter(logging.Formatter): 74 | LEVEL_MAP = {logging.FATAL: 'F', logging.ERROR: 'E', logging.WARN: 'W', logging.INFO: 'I', logging.DEBUG: 'D'} 75 | 76 | def format(self, record): 77 | record.levelletter = self.LEVEL_MAP[record.levelno] 78 | return super(CustomFormatter, self).format(record) 79 | 80 | 81 | def setup_logging(name=None): 82 | """Setup Google-Style logging for the entire application. 83 | 84 | At first I hated this but I had to use it for work, and now I prefer it. Who knew? 85 | From: https://github.com/twitter/commons/blob/master/src/python/twitter/common/log/formatters/glog.py 86 | 87 | Always logs DEBUG statements somewhere. 88 | 89 | Positional arguments: 90 | name -- Append this string to the log file filename. 91 | """ 92 | log_to_disk = False 93 | if OPTIONS['--log_dir']: 94 | if not os.path.isdir(OPTIONS['--log_dir']): 95 | print('ERROR: Directory {} does not exist.'.format(OPTIONS['--log_dir'])) 96 | sys.exit(1) 97 | if not os.access(OPTIONS['--log_dir'], os.W_OK): 98 | print('ERROR: No permissions to write to directory {}.'.format(OPTIONS['--log_dir'])) 99 | sys.exit(1) 100 | log_to_disk = True 101 | 102 | fmt = '%(levelletter)s%(asctime)s.%(msecs).03d %(process)d %(filename)s:%(lineno)d] %(message)s' 103 | datefmt = '%m%d %H:%M:%S' 104 | formatter = CustomFormatter(fmt, datefmt) 105 | 106 | console_handler = logging.StreamHandler(sys.stdout) 107 | console_handler.setLevel(logging.ERROR if log_to_disk else logging.DEBUG) 108 | console_handler.setFormatter(formatter) 109 | 110 | root = logging.getLogger() 111 | root.setLevel(logging.DEBUG) 112 | root.addHandler(console_handler) 113 | 114 | if log_to_disk: 115 | file_name = os.path.join(OPTIONS['--log_dir'], 'pypi_portal_{}.log'.format(name)) 116 | file_handler = logging.handlers.TimedRotatingFileHandler(file_name, when='d', backupCount=7) 117 | file_handler.setFormatter(formatter) 118 | root.addHandler(file_handler) 119 | 120 | 121 | def log_messages(app, port, fsh_folder): 122 | """Log messages common to Tornado and devserver.""" 123 | log = logging.getLogger(__name__) 124 | log.info('Server is running at http://0.0.0.0:{}/'.format(port)) 125 | log.info('Flask version: {}'.format(flask.__version__)) 126 | log.info('DEBUG: {}'.format(app.config['DEBUG'])) 127 | log.info('FLASK_STATICS_HELPER_FOLDER: {}'.format(fsh_folder)) 128 | log.info('STATIC_FOLDER: {}'.format(app.static_folder)) 129 | 130 | 131 | def parse_options(): 132 | """Parses command line options for Flask. 133 | 134 | Returns: 135 | Config instance to pass into create_app(). 136 | """ 137 | # Figure out which class will be imported. 138 | if OPTIONS['--config_prod']: 139 | config_class_string = 'pypi_portal.config.Production' 140 | else: 141 | config_class_string = 'pypi_portal.config.Config' 142 | config_obj = get_config(config_class_string) 143 | 144 | return config_obj 145 | 146 | 147 | def command(func): 148 | """Decorator that registers the chosen command/function. 149 | 150 | If a function is decorated with @command but that function name is not a valid "command" according to the docstring, 151 | a KeyError will be raised, since that's a bug in this script. 152 | 153 | If a user doesn't specify a valid command in their command line arguments, the above docopt(__doc__) line will print 154 | a short summary and call sys.exit() and stop up there. 155 | 156 | If a user specifies a valid command, but for some reason the developer did not register it, an AttributeError will 157 | raise, since it is a bug in this script. 158 | 159 | Finally, if a user specifies a valid command and it is registered with @command below, then that command is "chosen" 160 | by this decorator function, and set as the attribute `chosen`. It is then executed below in 161 | `if __name__ == '__main__':`. 162 | 163 | Doing this instead of using Flask-Script. 164 | 165 | Positional arguments: 166 | func -- the function to decorate 167 | """ 168 | @wraps(func) 169 | def wrapped(): 170 | return func() 171 | 172 | # Register chosen function. 173 | if func.__name__ not in OPTIONS: 174 | raise KeyError('Cannot register {}, not mentioned in docstring/docopt.'.format(func.__name__)) 175 | if OPTIONS[func.__name__]: 176 | command.chosen = func 177 | 178 | return wrapped 179 | 180 | 181 | @command 182 | def devserver(): 183 | setup_logging('devserver') 184 | app = create_app(parse_options()) 185 | fsh_folder = app.blueprints['flask_statics_helper'].static_folder 186 | log_messages(app, OPTIONS['--port'], fsh_folder) 187 | app.run(host='0.0.0.0', port=int(OPTIONS['--port'])) 188 | 189 | 190 | @command 191 | def tornadoserver(): 192 | setup_logging('tornadoserver') 193 | app = create_app(parse_options()) 194 | fsh_folder = app.blueprints['flask_statics_helper'].static_folder 195 | log_messages(app, OPTIONS['--port'], fsh_folder) 196 | 197 | # Setup the application. 198 | container = wsgi.WSGIContainer(app) 199 | application = web.Application([ 200 | (r'/static/flask_statics_helper/(.*)', web.StaticFileHandler, dict(path=fsh_folder)), 201 | (r'/(favicon\.ico)', web.StaticFileHandler, dict(path=app.static_folder)), 202 | (r'/static/(.*)', web.StaticFileHandler, dict(path=app.static_folder)), 203 | (r'.*', web.FallbackHandler, dict(fallback=container)) 204 | ]) # From http://maxburstein.com/blog/django-static-files-heroku/ 205 | http_server = httpserver.HTTPServer(application) 206 | http_server.bind(OPTIONS['--port']) 207 | 208 | # Start the server. 209 | http_server.start(0) # Forks multiple sub-processes 210 | ioloop.IOLoop.instance().start() 211 | 212 | 213 | @command 214 | def celerydev(): 215 | setup_logging('celerydev') 216 | app = create_app(parse_options(), no_sql=True) 217 | Logging._setup = True # Disable Celery from setting up logging, already done in setup_logging(). 218 | celery_args = ['celery', 'worker', '-B', '-s', '/tmp/celery.db', '--concurrency=5'] 219 | with app.app_context(): 220 | return celery_main(celery_args) 221 | 222 | 223 | @command 224 | def celerybeat(): 225 | setup_logging('celerybeat') 226 | app = create_app(parse_options(), no_sql=True) 227 | Logging._setup = True 228 | celery_args = ['celery', 'beat', '-C', '--pidfile', OPTIONS['--pid'], '-s', OPTIONS['--schedule']] 229 | with app.app_context(): 230 | return celery_main(celery_args) 231 | 232 | 233 | @command 234 | def celeryworker(): 235 | setup_logging('celeryworker{}'.format(OPTIONS['--name'])) 236 | app = create_app(parse_options(), no_sql=True) 237 | Logging._setup = True 238 | celery_args = ['celery', 'worker', '-n', OPTIONS['--name'], '-C', '--autoscale=10,1', '--without-gossip'] 239 | with app.app_context(): 240 | return celery_main(celery_args) 241 | 242 | 243 | @command 244 | def shell(): 245 | setup_logging('shell') 246 | app = create_app(parse_options()) 247 | app.app_context().push() 248 | Shell(make_context=lambda: dict(app=app, db=db)).run(no_ipython=False, no_bpython=False) 249 | 250 | 251 | @command 252 | def create_all(): 253 | setup_logging('create_all') 254 | app = create_app(parse_options()) 255 | log = logging.getLogger(__name__) 256 | with app.app_context(): 257 | tables_before = set(db.engine.table_names()) 258 | db.create_all() 259 | tables_after = set(db.engine.table_names()) 260 | created_tables = tables_after - tables_before 261 | for table in created_tables: 262 | log.info('Created table: {}'.format(table)) 263 | 264 | 265 | if __name__ == '__main__': 266 | signal.signal(signal.SIGINT, lambda *_: sys.exit(0)) # Properly handle Control+C 267 | if not OPTIONS['--port'].isdigit(): 268 | print('ERROR: Port should be a number.') 269 | sys.exit(1) 270 | getattr(command, 'chosen')() # Execute the function specified by the user. 271 | -------------------------------------------------------------------------------- /pypi_portal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Robpol86/Flask-Large-Application-Example/f536c3dee4050634cf4dfe77a6bb116040e6cf14/pypi_portal/__init__.py -------------------------------------------------------------------------------- /pypi_portal/application.py: -------------------------------------------------------------------------------- 1 | """Holds the create_app() Flask application factory. More information in create_app() docstring.""" 2 | 3 | from importlib import import_module 4 | import locale 5 | import os 6 | 7 | from flask import Flask 8 | from flask.ext.statics import Statics 9 | from yaml import load 10 | 11 | import pypi_portal as app_root 12 | from pypi_portal.blueprints import all_blueprints 13 | from pypi_portal.extensions import celery, db, mail, redis 14 | 15 | APP_ROOT_FOLDER = os.path.abspath(os.path.dirname(app_root.__file__)) 16 | TEMPLATE_FOLDER = os.path.join(APP_ROOT_FOLDER, 'templates') 17 | STATIC_FOLDER = os.path.join(APP_ROOT_FOLDER, 'static') 18 | REDIS_SCRIPTS_FOLDER = os.path.join(APP_ROOT_FOLDER, 'redis_scripts') 19 | 20 | 21 | def get_config(config_class_string, yaml_files=None): 22 | """Load the Flask config from a class. 23 | 24 | Positional arguments: 25 | config_class_string -- string representation of a configuration class that will be loaded (e.g. 26 | 'pypi_portal.config.Production'). 27 | yaml_files -- List of YAML files to load. This is for testing, leave None in dev/production. 28 | 29 | Returns: 30 | A class object to be fed into app.config.from_object(). 31 | """ 32 | config_module, config_class = config_class_string.rsplit('.', 1) 33 | config_class_object = getattr(import_module(config_module), config_class) 34 | config_obj = config_class_object() 35 | 36 | # Expand some options. 37 | celery_fmt = 'pypi_portal.tasks.{}' 38 | db_fmt = 'pypi_portal.models.{}' 39 | if getattr(config_obj, 'CELERY_IMPORTS', False): 40 | config_obj.CELERY_IMPORTS = [celery_fmt.format(m) for m in config_obj.CELERY_IMPORTS] 41 | for definition in getattr(config_obj, 'CELERYBEAT_SCHEDULE', dict()).values(): 42 | definition.update(task=celery_fmt.format(definition['task'])) 43 | if getattr(config_obj, 'DB_MODELS_IMPORTS', False): 44 | config_obj.DB_MODELS_IMPORTS = [db_fmt.format(m) for m in config_obj.DB_MODELS_IMPORTS] 45 | #for script_name, script_file in getattr(config_obj, 'REDIS_SCRIPTS', dict()).items(): 46 | # config_obj.REDIS_SCRIPTS[script_name] = os.path.join(REDIS_SCRIPTS_FOLDER, script_file) 47 | 48 | # Load additional configuration settings. 49 | yaml_files = yaml_files or [f for f in [ 50 | os.path.join('/', 'etc', 'pypi_portal', 'config.yml'), 51 | os.path.abspath(os.path.join(APP_ROOT_FOLDER, '..', 'config.yml')), 52 | os.path.join(APP_ROOT_FOLDER, 'config.yml'), 53 | ] if os.path.exists(f)] 54 | additional_dict = dict() 55 | for y in yaml_files: 56 | with open(y) as f: 57 | additional_dict.update(load(f.read())) 58 | 59 | # Merge the rest into the Flask app config. 60 | for key, value in additional_dict.items(): 61 | setattr(config_obj, key, value) 62 | 63 | return config_obj 64 | 65 | 66 | def create_app(config_obj, no_sql=False): 67 | """Flask application factory. Initializes and returns the Flask application. 68 | 69 | Blueprints are registered in here. 70 | 71 | Modeled after: http://flask.pocoo.org/docs/patterns/appfactories/ 72 | 73 | Positional arguments: 74 | config_obj -- configuration object to load into app.config. 75 | 76 | Keyword arguments: 77 | no_sql -- does not run init_app() for the SQLAlchemy instance. For Celery compatibility. 78 | 79 | Returns: 80 | The initialized Flask application. 81 | """ 82 | # Initialize app. Flatten config_obj to dictionary (resolve properties). 83 | app = Flask(__name__, template_folder=TEMPLATE_FOLDER, static_folder=STATIC_FOLDER) 84 | config_dict = dict([(k, getattr(config_obj, k)) for k in dir(config_obj) if not k.startswith('_')]) 85 | app.config.update(config_dict) 86 | 87 | # Import DB models. Flask-SQLAlchemy doesn't do this automatically like Celery does. 88 | with app.app_context(): 89 | for module in app.config.get('DB_MODELS_IMPORTS', list()): 90 | import_module(module) 91 | 92 | # Setup redirects and register blueprints. 93 | app.add_url_rule('/favicon.ico', 'favicon', lambda: app.send_static_file('favicon.ico')) 94 | for bp in all_blueprints: 95 | import_module(bp.import_name) 96 | app.register_blueprint(bp) 97 | 98 | # Initialize extensions/add-ons/plugins. 99 | if not no_sql: 100 | db.init_app(app) 101 | Statics(app) # Enable Flask-Statics-Helper features. 102 | redis.init_app(app) 103 | celery.init_app(app) 104 | mail.init_app(app) 105 | 106 | # Activate middleware. 107 | locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') # For filters inside the middleware file. 108 | with app.app_context(): 109 | import_module('pypi_portal.middleware') 110 | 111 | # Return the application instance. 112 | return app 113 | -------------------------------------------------------------------------------- /pypi_portal/blueprints.py: -------------------------------------------------------------------------------- 1 | """All Flask blueprints for the entire application. 2 | 3 | All blueprints for all views go here. They shall be imported by the views themselves and by application.py. Blueprint 4 | URL paths are defined here as well. 5 | """ 6 | 7 | from flask import Blueprint 8 | 9 | 10 | def _factory(partial_module_string, url_prefix): 11 | """Generates blueprint objects for view modules. 12 | 13 | Positional arguments: 14 | partial_module_string -- string representing a view module without the absolute path (e.g. 'home.index' for 15 | pypi_portal.views.home.index). 16 | url_prefix -- URL prefix passed to the blueprint. 17 | 18 | Returns: 19 | Blueprint instance for a view module. 20 | """ 21 | name = partial_module_string 22 | import_name = 'pypi_portal.views.{}'.format(partial_module_string) 23 | template_folder = 'templates' 24 | blueprint = Blueprint(name, import_name, template_folder=template_folder, url_prefix=url_prefix) 25 | return blueprint 26 | 27 | 28 | examples_alerts = _factory('examples.alerts', '/examples/alerts') 29 | examples_exception = _factory('examples.exception', '/examples/exception') 30 | home_index = _factory('home.index', '/') 31 | pypi_packages = _factory('pypi.packages', '/pypi') 32 | 33 | 34 | all_blueprints = (examples_alerts, examples_exception, home_index, pypi_packages,) 35 | -------------------------------------------------------------------------------- /pypi_portal/config.py: -------------------------------------------------------------------------------- 1 | from urllib import quote_plus 2 | from celery.schedules import crontab 3 | 4 | 5 | class HardCoded(object): 6 | """Constants used throughout the application. 7 | 8 | All hard coded settings/data that are not actual/official configuration options for Flask, Celery, or their 9 | extensions goes here. 10 | """ 11 | ADMINS = ['me@me.test'] 12 | DB_MODELS_IMPORTS = ('pypi',) # Like CELERY_IMPORTS in CeleryConfig. 13 | ENVIRONMENT = property(lambda self: self.__class__.__name__) 14 | MAIL_EXCEPTION_THROTTLE = 24 * 60 * 60 15 | _SQLALCHEMY_DATABASE_DATABASE = 'pypi_portal' 16 | _SQLALCHEMY_DATABASE_HOSTNAME = 'localhost' 17 | _SQLALCHEMY_DATABASE_PASSWORD = 'pypi_p@ssword' 18 | _SQLALCHEMY_DATABASE_USERNAME = 'pypi_service' 19 | 20 | 21 | class CeleryConfig(HardCoded): 22 | """Configurations used by Celery only.""" 23 | CELERYD_PREFETCH_MULTIPLIER = 1 24 | CELERYD_TASK_SOFT_TIME_LIMIT = 20 * 60 # Raise exception if task takes too long. 25 | CELERYD_TASK_TIME_LIMIT = 30 * 60 # Kill worker if task takes way too long. 26 | CELERY_ACCEPT_CONTENT = ['json'] 27 | CELERY_ACKS_LATE = True 28 | CELERY_DISABLE_RATE_LIMITS = True 29 | CELERY_IMPORTS = ('pypi',) 30 | CELERY_RESULT_SERIALIZER = 'json' 31 | CELERY_TASK_RESULT_EXPIRES = 10 * 60 # Dispose of Celery Beat results after 10 minutes. 32 | CELERY_TASK_SERIALIZER = 'json' 33 | CELERY_TRACK_STARTED = True 34 | 35 | CELERYBEAT_SCHEDULE = { 36 | 'pypy-every-day': dict(task='pypi.update_package_list', schedule=crontab(hour='0')), 37 | } 38 | 39 | 40 | class Config(CeleryConfig): 41 | """Default Flask configuration inherited by all environments. Use this for development environments.""" 42 | DEBUG = True 43 | TESTING = False 44 | SECRET_KEY = "i_don't_want_my_cookies_expiring_while_developing" 45 | MAIL_SERVER = 'smtp.localhost.test' 46 | MAIL_DEFAULT_SENDER = 'admin@demo.test' 47 | MAIL_SUPPRESS_SEND = True 48 | REDIS_URL = 'redis://localhost/0' 49 | SQLALCHEMY_DATABASE_URI = property(lambda self: 'mysql://{u}:{p}@{h}/{d}'.format( 50 | d=quote_plus(self._SQLALCHEMY_DATABASE_DATABASE), h=quote_plus(self._SQLALCHEMY_DATABASE_HOSTNAME), 51 | p=quote_plus(self._SQLALCHEMY_DATABASE_PASSWORD), u=quote_plus(self._SQLALCHEMY_DATABASE_USERNAME) 52 | )) 53 | 54 | 55 | class Testing(Config): 56 | TESTING = True 57 | CELERY_ALWAYS_EAGER = True 58 | REDIS_URL = 'redis://localhost/1' 59 | _SQLALCHEMY_DATABASE_DATABASE = 'pypi_portal_testing' 60 | 61 | 62 | class Production(Config): 63 | DEBUG = False 64 | SECRET_KEY = None # To be overwritten by a YAML file. 65 | ADMINS = ['my-team@me.test'] 66 | MAIL_SUPPRESS_SEND = False 67 | STATICS_MINIFY = True 68 | -------------------------------------------------------------------------------- /pypi_portal/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Robpol86/Flask-Large-Application-Example/f536c3dee4050634cf4dfe77a6bb116040e6cf14/pypi_portal/core/__init__.py -------------------------------------------------------------------------------- /pypi_portal/core/email.py: -------------------------------------------------------------------------------- 1 | """Convenience functions for sending email from any view or Celery task.""" 2 | 3 | from contextlib import contextmanager 4 | import hashlib 5 | from logging import getLogger 6 | 7 | from flask import current_app 8 | from flask.ext.mail import Message 9 | from werkzeug.debug import tbtools 10 | 11 | from pypi_portal.extensions import mail, redis 12 | from pypi_portal.models.redis import EMAIL_THROTTLE 13 | 14 | LOG = getLogger(__name__) 15 | 16 | 17 | @contextmanager 18 | def _override_html(): 19 | """Temporarily changes the module constants in `tbtools` to make it email-friendly. 20 | 21 | Gmail strips out everything between , so all styling has to be inline using the style="" attribute in 22 | HTML tags. These changes makes the Flask debugging page HTML (shown when unhandled exceptions are raised with 23 | DEBUG = True) email-friendly. Designed to be used with the `with` statement. 24 | 25 | It's too bad `tbtools.Traceback` doesn't copy module constants to instance variables where they can be easily 26 | overridden, ugh! 27 | """ 28 | # Backup. 29 | old_page_html = tbtools.PAGE_HTML 30 | old_summary = tbtools.SUMMARY_HTML 31 | old_frame = tbtools.FRAME_HTML 32 | # Get new HTML. 33 | email_template = current_app.jinja_env.get_template('email.html') 34 | email_context = email_template.new_context() 35 | page_html = email_template.blocks['page_html'](email_context).next() 36 | summary_html = email_template.blocks['summary_html'](email_context).next() 37 | frame_html = email_template.blocks['frame_html'](email_context).next() 38 | # Change module variables. 39 | tbtools.PAGE_HTML = page_html 40 | tbtools.SUMMARY_HTML = summary_html 41 | tbtools.FRAME_HTML = frame_html 42 | yield # Let `with` block execute. 43 | # Revert changes. 44 | tbtools.PAGE_HTML = old_page_html 45 | tbtools.SUMMARY_HTML = old_summary 46 | tbtools.FRAME_HTML = old_frame 47 | 48 | 49 | def send_exception(subject): 50 | """Send Python exception tracebacks via email to the ADMINS list. 51 | 52 | Use the same HTML styling as Flask tracebacks in debug web servers. 53 | 54 | This function must be called while the exception is happening. It picks up the raised exception with sys.exc_info(). 55 | 56 | Positional arguments: 57 | subject -- subject line of the email (to be prepended by 'Application Error: '). 58 | """ 59 | # Generate and modify html. 60 | tb = tbtools.get_current_traceback() # Get exception information. 61 | with _override_html(): 62 | html = tb.render_full().encode('utf-8', 'replace') 63 | html = html.replace('
', '
') 64 | subject = 'Application Error: {}'.format(subject) 65 | 66 | # Apply throttle. 67 | md5 = hashlib.md5('{}{}'.format(subject, html)).hexdigest() 68 | seconds = int(current_app.config['MAIL_EXCEPTION_THROTTLE']) 69 | lock = redis.lock(EMAIL_THROTTLE.format(md5=md5), timeout=seconds) 70 | have_lock = lock.acquire(blocking=False) 71 | if not have_lock: 72 | LOG.debug('Suppressing email: {}'.format(subject)) 73 | return 74 | 75 | # Send email. 76 | msg = Message(subject=subject, recipients=current_app.config['ADMINS'], html=html) 77 | mail.send(msg) 78 | 79 | 80 | def send_email(subject, body=None, html=None, recipients=None, throttle=None): 81 | """Send an email. Optionally throttle the amount an identical email goes out. 82 | 83 | If the throttle argument is set, an md5 checksum derived from the subject, body, html, and recipients is stored in 84 | Redis with a lock timeout. On the first email sent, the email goes out like normal. But when other emails with the 85 | same subject, body, html, and recipients is supposed to go out, and the lock hasn't expired yet, the email will be 86 | dropped and never sent. 87 | 88 | Positional arguments: 89 | subject -- the subject line of the email. 90 | 91 | Keyword arguments. 92 | body -- the body of the email (no HTML). 93 | html -- the body of the email, can be HTML (overrides body). 94 | recipients -- list or set (not string) of email addresses to send the email to. Defaults to the ADMINS list in the 95 | Flask config. 96 | throttle -- time in seconds or datetime.timedelta object between sending identical emails. 97 | """ 98 | recipients = recipients or current_app.config['ADMINS'] 99 | if throttle is not None: 100 | md5 = hashlib.md5('{}{}{}{}'.format(subject, body, html, recipients)).hexdigest() 101 | seconds = throttle.total_seconds() if hasattr(throttle, 'total_seconds') else throttle 102 | lock = redis.lock(EMAIL_THROTTLE.format(md5=md5), timeout=int(seconds)) 103 | have_lock = lock.acquire(blocking=False) 104 | if not have_lock: 105 | LOG.debug('Suppressing email: {}'.format(subject)) 106 | return 107 | msg = Message(subject=subject, recipients=recipients, body=body, html=html) 108 | mail.send(msg) 109 | -------------------------------------------------------------------------------- /pypi_portal/core/flash.py: -------------------------------------------------------------------------------- 1 | """Convenience wrappers for flask.flash() with special-character handling. 2 | 3 | With PyCharm inspections it's easy to see which custom flash messages are available. If you directly use flask.flash(), 4 | the "type" of message (info, warning, etc.) is a string passed as a second argument to the function. With this file 5 | PyCharm will tell you which type of messages are supported. 6 | """ 7 | 8 | from flask import flash 9 | 10 | 11 | def _escape(message): 12 | """Escape some characters in a message. Make them HTML friendly. 13 | 14 | Positional arguments: 15 | message -- the string to process. 16 | 17 | Returns: 18 | Escaped string. 19 | """ 20 | translations = { 21 | '"': '"', 22 | "'": ''', 23 | '`': '‘', 24 | '\n': '
', 25 | } 26 | for k, v in translations.items(): 27 | message = message.replace(k, v) 28 | 29 | return message 30 | 31 | 32 | def default(message): 33 | return flash(_escape(message), 'default') 34 | 35 | 36 | def success(message): 37 | return flash(_escape(message), 'success') 38 | 39 | 40 | def info(message): 41 | return flash(_escape(message), 'info') 42 | 43 | 44 | def warning(message): 45 | return flash(_escape(message), 'warning') 46 | 47 | 48 | def danger(message): 49 | return flash(_escape(message), 'danger') 50 | 51 | 52 | def well(message): 53 | return flash(_escape(message), 'well') 54 | 55 | 56 | def modal(message): 57 | return flash(_escape(message), 'modal') 58 | -------------------------------------------------------------------------------- /pypi_portal/extensions.py: -------------------------------------------------------------------------------- 1 | """Flask and other extensions instantiated here. 2 | 3 | To avoid circular imports with views and create_app(), extensions are instantiated here. They will be initialized 4 | (calling init_app()) in application.py. 5 | """ 6 | 7 | from logging import getLogger 8 | 9 | from flask.ext.celery import Celery 10 | from flask.ext.mail import Mail 11 | from flask.ext.redis import Redis 12 | from flask.ext.sqlalchemy import SQLAlchemy 13 | from sqlalchemy.event import listens_for 14 | from sqlalchemy.pool import Pool 15 | 16 | LOG = getLogger(__name__) 17 | 18 | 19 | @listens_for(Pool, 'connect', named=True) 20 | def _on_connect(dbapi_connection, **_): 21 | """Set MySQL mode to TRADITIONAL on databases that don't set this automatically. 22 | 23 | Without this, MySQL will silently insert invalid values in the database, causing very long debugging sessions in the 24 | long run. 25 | http://www.enricozini.org/2012/tips/sa-sqlmode-traditional/ 26 | """ 27 | LOG.debug('Setting SQL Mode to TRADITIONAL.') 28 | dbapi_connection.cursor().execute("SET SESSION sql_mode='TRADITIONAL'") 29 | 30 | 31 | celery = Celery() 32 | db = SQLAlchemy() 33 | mail = Mail() 34 | redis = Redis() 35 | -------------------------------------------------------------------------------- /pypi_portal/middleware.py: -------------------------------------------------------------------------------- 1 | """Flask middleware definitions. This is also where template filters are defined. 2 | 3 | To be imported by the application.current_app() factory. 4 | """ 5 | 6 | import locale 7 | from logging import getLogger 8 | import os 9 | 10 | from celery.signals import task_failure, worker_process_init 11 | from flask import current_app, render_template, request 12 | from markupsafe import Markup 13 | 14 | from pypi_portal.core.email import send_exception 15 | from pypi_portal.extensions import db 16 | 17 | LOG = getLogger(__name__) 18 | 19 | 20 | # Fix Flask-SQLAlchemy and Celery incompatibilities. 21 | @worker_process_init.connect 22 | def celery_worker_init_db(**_): 23 | """Initialize SQLAlchemy right after the Celery worker process forks. 24 | 25 | This ensures each Celery worker has its own dedicated connection to the MySQL database. Otherwise 26 | one worker may close the connection while another worker is using it, raising exceptions. 27 | 28 | Without this, the existing session to the MySQL server is cloned to all Celery workers, so they 29 | all share a single session. A SQLAlchemy session is not thread/concurrency-safe, causing weird 30 | exceptions to be raised by workers. 31 | 32 | Based on http://stackoverflow.com/a/14146403/1198943 33 | """ 34 | LOG.debug('Initializing SQLAlchemy for PID {}.'.format(os.getpid())) 35 | db.init_app(current_app) 36 | 37 | 38 | # Send email when a Celery task raises an unhandled exception. 39 | @task_failure.connect 40 | def celery_error_handler(sender, exception, **_): 41 | exception_name = exception.__class__.__name__ 42 | task_module = sender.name 43 | send_exception('{} exception in {}'.format(exception_name, task_module)) 44 | 45 | 46 | # Setup default error templates. 47 | @current_app.errorhandler(400) 48 | @current_app.errorhandler(403) 49 | @current_app.errorhandler(404) 50 | @current_app.errorhandler(500) 51 | def error_handler(e): 52 | code = getattr(e, 'code', 500) # If 500, e == the exception. 53 | if code == 500: 54 | # Send email to all ADMINS. 55 | exception_name = e.__class__.__name__ 56 | view_module = request.endpoint 57 | send_exception('{} exception in {}'.format(exception_name, view_module)) 58 | return render_template('{}.html'.format(code)), code 59 | 60 | 61 | # Template filters. 62 | @current_app.template_filter() 63 | def whitelist(value): 64 | """Whitelist specific HTML tags and strings. 65 | 66 | Positional arguments: 67 | value -- the string to perform the operation on. 68 | 69 | Returns: 70 | Markup() instance, indicating the string is safe. 71 | """ 72 | translations = { 73 | '&quot;': '"', 74 | '&#39;': ''', 75 | '&lsquo;': '‘', 76 | '&nbsp;': ' ', 77 | '<br>': '
', 78 | } 79 | escaped = str(Markup.escape(value)) # Escapes everything. 80 | for k, v in translations.items(): 81 | escaped = escaped.replace(k, v) # Un-escape specific elements using str.replace. 82 | return Markup(escaped) # Return as 'safe'. 83 | 84 | 85 | @current_app.template_filter() 86 | def dollar(value): 87 | """Formats the float value into two-decimal-points dollar amount. 88 | From http://flask.pocoo.org/docs/templating/ 89 | 90 | Positional arguments: 91 | value -- the string representation of a float to perform the operation on. 92 | 93 | Returns: 94 | Dollar formatted string. 95 | """ 96 | return locale.currency(float(value), grouping=True) 97 | 98 | 99 | @current_app.template_filter() 100 | def sum_key(value, key): 101 | """Sums up the numbers in a 'column' in a list of dictionaries or objects. 102 | 103 | Positional arguments: 104 | value -- list of dictionaries or objects to iterate through. 105 | 106 | Returns: 107 | Sum of the values. 108 | """ 109 | values = [r.get(key, 0) if hasattr(r, 'get') else getattr(r, key, 0) for r in value] 110 | return sum(values) 111 | 112 | 113 | @current_app.template_filter() 114 | def max_key(value, key): 115 | """Returns the maximum value in a 'column' in a list of dictionaries or objects. 116 | 117 | Positional arguments: 118 | value -- list of dictionaries or objects to iterate through. 119 | 120 | Returns: 121 | Sum of the values. 122 | """ 123 | values = [r.get(key, 0) if hasattr(r, 'get') else getattr(r, key, 0) for r in value] 124 | return max(values) 125 | 126 | 127 | @current_app.template_filter() 128 | def average_key(value, key): 129 | """Returns the average value in a 'column' in a list of dictionaries or objects. 130 | 131 | Positional arguments: 132 | value -- list of dictionaries or objects to iterate through. 133 | 134 | Returns: 135 | Sum of the values. 136 | """ 137 | values = [r.get(key, 0) if hasattr(r, 'get') else getattr(r, key, 0) for r in value] 138 | return float(sum(values)) / (len(values) or float('nan')) 139 | -------------------------------------------------------------------------------- /pypi_portal/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Robpol86/Flask-Large-Application-Example/f536c3dee4050634cf4dfe77a6bb116040e6cf14/pypi_portal/models/__init__.py -------------------------------------------------------------------------------- /pypi_portal/models/helpers.py: -------------------------------------------------------------------------------- 1 | """Convenience functions which interact with SQLAlchemy models.""" 2 | 3 | from sqlalchemy import Column, func, Integer 4 | from sqlalchemy.ext.declarative import declared_attr 5 | 6 | from pypi_portal.extensions import db 7 | 8 | 9 | class Base(db.Model): 10 | """Convenience base DB model class. Makes sure tables in MySQL are created as InnoDB. 11 | 12 | This is to enforce foreign key constraints (MyISAM doesn't support constraints) outside of production. Tables are 13 | also named to avoid collisions. 14 | """ 15 | 16 | @declared_attr 17 | def __tablename__(self): 18 | return '{}_{}'.format(self.__module__.split('.')[-1], self.__name__.lower()) 19 | 20 | __abstract__ = True 21 | __table_args__ = dict(mysql_charset='utf8', mysql_engine='InnoDB') 22 | id = Column(Integer, primary_key=True, autoincrement=True, nullable=True) 23 | 24 | 25 | def count(column, value, glob=False): 26 | """Counts number of rows with value in a column. This function is case-insensitive. 27 | 28 | Positional arguments: 29 | column -- the SQLAlchemy column object to search in (e.g. Table.a_column). 30 | value -- the value to search for, any string. 31 | 32 | Keyword arguments: 33 | glob -- enable %globbing% search (default False). 34 | 35 | Returns: 36 | Number of rows that match. Equivalent of SELECT count(*) FROM. 37 | """ 38 | query = db.session.query(func.count('*')) 39 | if glob: 40 | query = query.filter(column.ilike(value)) 41 | else: 42 | query = query.filter(func.lower(column) == value.lower()) 43 | return query.one()[0] 44 | -------------------------------------------------------------------------------- /pypi_portal/models/pypi.py: -------------------------------------------------------------------------------- 1 | """Holds all data cached from PyPI.""" 2 | 3 | from sqlalchemy import Column, String, Text 4 | 5 | from pypi_portal.models.helpers import Base 6 | 7 | 8 | class Package(Base): 9 | """Holds all of the packages on PyPI.""" 10 | name = Column(String(255), unique=True, nullable=False) 11 | summary = Column(Text, nullable=False) 12 | latest_version = Column(String(128), nullable=False) 13 | -------------------------------------------------------------------------------- /pypi_portal/models/redis.py: -------------------------------------------------------------------------------- 1 | """Redis keys used throughout the entire application (Flask, etc.).""" 2 | 3 | # Email throttling. 4 | EMAIL_THROTTLE = 'pypi_portal:email_throttle:{md5}' # Lock. 5 | 6 | # PyPI throttling. 7 | POLL_SIMPLE_THROTTLE = 'pypi_portal:poll_simple_throttle' # Lock. 8 | -------------------------------------------------------------------------------- /pypi_portal/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Robpol86/Flask-Large-Application-Example/f536c3dee4050634cf4dfe77a6bb116040e6cf14/pypi_portal/static/favicon.ico -------------------------------------------------------------------------------- /pypi_portal/static/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 21 | 23 | 26 | 30 | 34 | 35 | 44 | 47 | 51 | 55 | 56 | 65 | 66 | 84 | 86 | 87 | 89 | image/svg+xml 90 | 92 | 93 | 94 | 95 | 100 | 103 | 107 | 111 | 112 | 113 | 114 | -------------------------------------------------------------------------------- /pypi_portal/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Robpol86/Flask-Large-Application-Example/f536c3dee4050634cf4dfe77a6bb116040e6cf14/pypi_portal/tasks/__init__.py -------------------------------------------------------------------------------- /pypi_portal/tasks/pypi.py: -------------------------------------------------------------------------------- 1 | """Retrieve data from PyPI.""" 2 | 3 | from distutils.version import LooseVersion 4 | from logging import getLogger 5 | import xmlrpclib 6 | 7 | from flask.ext.celery import single_instance 8 | 9 | from pypi_portal.extensions import celery, db, redis 10 | from pypi_portal.models.pypi import Package 11 | from pypi_portal.models.redis import POLL_SIMPLE_THROTTLE 12 | 13 | LOG = getLogger(__name__) 14 | THROTTLE = 1 * 60 * 60 15 | 16 | 17 | @celery.task(bind=True, soft_time_limit=120) 18 | @single_instance 19 | def update_package_list(): 20 | """Get a list of all packages from PyPI through their XMLRPC API. 21 | 22 | This task returns something in case the user schedules it from a view. The view can wait up to a certain amount of 23 | time for this task to finish, and if nothing times out, it can tell the user if it found any new packages. 24 | 25 | Since views can schedule this task, we don't want some rude person hammering PyPI or our application with repeated 26 | requests. This task is limited to one run per 1 hour at most. 27 | 28 | Returns: 29 | List of new packages found. Returns None if task is rate-limited. 30 | """ 31 | # Rate limit. 32 | lock = redis.lock(POLL_SIMPLE_THROTTLE, timeout=int(THROTTLE)) 33 | have_lock = lock.acquire(blocking=False) 34 | if not have_lock: 35 | LOG.warning('poll_simple() task has already executed in the past 4 hours. Rate limiting.') 36 | return None 37 | 38 | # Query API. 39 | client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi') 40 | results = client.search(dict(summary='')) 41 | if not results: 42 | LOG.error('Reply from API had no results.') 43 | return list() 44 | 45 | LOG.debug('Sorting results.') 46 | results.sort(key=lambda x: (x['name'], LooseVersion(x['version']))) 47 | filtered = (r for r in results if r['version'][0].isdigit()) 48 | packages = {r['name']: dict(summary=r['summary'], version=r['version'], id=0) for r in filtered} 49 | 50 | LOG.debug('Pruning unchanged packages.') 51 | for row in db.session.query(Package.id, Package.name, Package.summary, Package.latest_version): 52 | if packages.get(row[1]) == dict(summary=row[2], version=row[3], id=0): 53 | packages.pop(row[1]) 54 | elif row[1] in packages: 55 | packages[row[1]]['id'] = row[0] 56 | new_package_names = {n for n, d in packages.items() if not d['id']} 57 | 58 | # Merge into database. 59 | LOG.debug('Found {} new packages in PyPI, updating {} total.'.format(len(new_package_names), len(packages))) 60 | with db.session.begin_nested(): 61 | for name, data in packages.items(): 62 | db.session.merge(Package(id=data['id'], name=name, summary=data['summary'], latest_version=data['version'])) 63 | db.session.commit() 64 | return list(new_package_names) 65 | -------------------------------------------------------------------------------- /pypi_portal/templates/400.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 400{% endblock %} 4 | 5 | {% block container %} 6 |
7 |

400 Bad Request

8 |

Page found but the POST/GET data sent was invalid.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /pypi_portal/templates/403.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 403{% endblock %} 4 | 5 | {% block container %} 6 |
7 |

403 Forbidden

8 |

Access denied.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /pypi_portal/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 404{% endblock %} 4 | 5 | {% block container %} 6 |
7 |

404 Not Found

8 |

Page not found.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /pypi_portal/templates/500.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 500{% endblock %} 4 | 5 | {% block container %} 6 |
7 |

500 Internal Server Error

8 |

Uh-oh you found a bug! An email has been sent to the ADMINs list.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /pypi_portal/templates/base.html: -------------------------------------------------------------------------------- 1 | {% extends 'flask_statics_helper/bootstrap.html' %} 2 | {% from 'flash.html' import normal_alert %} 3 | 4 | {% set flash_messages_normal = get_flashed_messages(with_categories=True, 5 | category_filter=['default', 'success', 'info', 'warning', 'danger']) %} 6 | {% set flash_messages_modal = get_flashed_messages(category_filter=['modal']) %} 7 | {% set flash_messages_well = get_flashed_messages(category_filter=['well']) %} 8 | 9 | {% if flash_messages_normal %}{% set STATICS_ENABLE_RESOURCE_BOOTSTRAP_GROWL = True %}{% endif %} 10 | {% if flash_messages_well %}{% set STATICS_ENABLE_RESOURCE_CSSHAKE = True %}{% endif %} 11 | 12 | {% block title %}PyPI Portal{%- block append_title %}{% endblock %}{% endblock %} 13 | 14 | {% block styles %} 15 | {{ super() }} 16 | 17 | 18 | {% if flash_messages_normal or flash_messages_modal or flash_messages_well %} 19 | 22 | {% endif %} 23 | {% endblock %} 24 | 25 | {% block navbar %} 26 | {# Need to set these variables to fix a "context depth limit" with Jinja2 blocks and includes. #} 27 | {% set flash_messages_modal = flash_messages_modal %} 28 | {% set flash_messages_well = flash_messages_well %} 29 | {% include 'navbar.html' with context %} 30 | {% endblock %} 31 | 32 | {% block scripts %} 33 | {{ super() }} 34 | 35 | {% if flash_messages_normal %} 36 | 42 | {% endif %} 43 | 44 | {{ normal_alert(flash_messages_normal) }} 45 | {% endblock %} 46 | 47 | {# Includes jQuery already. Specify additional JS libraries like d3/FontAwesome in your template. #} 48 | -------------------------------------------------------------------------------- /pypi_portal/templates/email.html: -------------------------------------------------------------------------------- 1 | {# This template is only used by pypi_portal.core.email. #} 2 | 3 | {% block page_html %} 4 | 5 | 6 | %(title)s // Werkzeug Debugger 7 | 8 |
9 |

%(exception_type)s

10 |

%(exception)s

11 |

13 | Traceback 14 | (most recent call last) 15 |

16 | %(summary)s 17 |
18 | The debugger caught an exception in your WSGI application. You can now look at the traceback which led to 19 | the error. 20 |
21 |
22 | Brought to you by DON'T PANIC, your friendly Werkzeug powered traceback interpreter. 23 |
24 |
25 | 26 | 27 | {% endblock %} 28 | 29 | {% block summary_html %} 30 |
31 | %(title)s 32 |
    %(frames)s
33 | %(description)s 34 |
35 | {% endblock %} 36 | 37 | {% block frame_html %} 38 |
39 |

40 | File "%(filename)s", line %(lineno)s, 41 | in %(function_name)s 42 |

43 |
45 |         %(current_line)s
46 |     
47 |
48 | {% endblock %} 49 | -------------------------------------------------------------------------------- /pypi_portal/templates/flash.html: -------------------------------------------------------------------------------- 1 | {########################################## Well Alerts ###########################################} 2 | {% macro well_alert_navbar(messages) %}{% if messages %} 3 | 11 | {% endif %}{% endmacro %} 12 | 13 | {% macro well_alert(messages) %}{% if messages %} 14 |
15 | {% for message in messages %} 16 |
17 | {{ message|whitelist }} 18 |
19 | {% endfor %} 20 |
21 | {% endif %}{% endmacro %} 22 | 23 | 24 | {########################################## Modal Alerts ##########################################} 25 | {% macro modal_alert(messages) %}{% if messages %} 26 | 44 | {% endif %}{% endmacro %} 45 | 46 | 47 | {##################################### Normal (Growl) Alerts ######################################} 48 | {% macro normal_alert(messages) %}{% if messages %} 49 | 54 | {% endif %}{% endmacro %} 55 | -------------------------------------------------------------------------------- /pypi_portal/templates/navbar.html: -------------------------------------------------------------------------------- 1 | {% from 'flash.html' import modal_alert, well_alert_navbar, well_alert %} 2 | 3 | 36 | {{ well_alert(flash_messages_well) }} 37 | {{ modal_alert(flash_messages_modal) }} 38 | -------------------------------------------------------------------------------- /pypi_portal/views/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Robpol86/Flask-Large-Application-Example/f536c3dee4050634cf4dfe77a6bb116040e6cf14/pypi_portal/views/__init__.py -------------------------------------------------------------------------------- /pypi_portal/views/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Robpol86/Flask-Large-Application-Example/f536c3dee4050634cf4dfe77a6bb116040e6cf14/pypi_portal/views/examples/__init__.py -------------------------------------------------------------------------------- /pypi_portal/views/examples/alerts.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | from flask import abort, redirect, render_template, request, url_for 4 | 5 | from pypi_portal.core import flash 6 | from pypi_portal.blueprints import examples_alerts 7 | 8 | 9 | @examples_alerts.route('/') 10 | def index(): 11 | return render_template('examples_alerts.html') 12 | 13 | 14 | @examples_alerts.route('/modal') 15 | def modal(): 16 | """Push flash message to stack, then redirect back to index().""" 17 | message_size = request.args.get('message_size') 18 | flash_count = request.args.get('flash_count') 19 | flash_type = request.args.get('flash_type') 20 | 21 | # First check if requested type/count are valid. 22 | available_types = [k for k, v in flash.__dict__.items() if callable(v)] 23 | if flash_type not in available_types: 24 | abort(400) 25 | if not str(flash_count).isdigit() or not (1 <= int(flash_count) <= 10): 26 | abort(400) 27 | 28 | # Build message. 29 | if message_size == 'large': 30 | message = dedent("""\ 31 | Traceback (most recent call last): 32 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/tornado/web.py", line 1309, in _execute 33 | result = self.prepare() 34 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/tornado/web.py", line 2498, in prepare 35 | self.fallback(self.request) 36 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/tornado/wsgi.py", line 280, in __call__ 37 | WSGIContainer.environ(request), start_response) 38 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/flask/app.py", line 1836, in __call__ 39 | return self.wsgi_app(environ, start_response) 40 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/flask/app.py", line 1820, in wsgi_app 41 | response = self.make_response(self.handle_exception(e)) 42 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/flask/app.py", line 1410, in handle_exception 43 | return handler(e) 44 | File "/Users/robpol86/workspace/Flask-Large-Application-Example/pypi_portal/middleware.py", line 56, in error_handler 45 | send_exception('{} exception in {}'.format(exception_name, view_module)) 46 | File "/Users/robpol86/workspace/Flask-Large-Application-Example/pypi_portal/core/email.py", line 77, in send_exception 47 | mail.send(msg) 48 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/flask_mail.py", line 415, in send 49 | with self.connect() as connection: 50 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/flask_mail.py", line 123, in __enter__ 51 | self.host = self.configure_host() 52 | File "/Users/robpol86/virtualenvs/Flask-Large-App/lib/python2.7/site-packages/flask_mail.py", line 137, in configure_host 53 | host = smtplib.SMTP(self.mail.server, self.mail.port) 54 | File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/smtplib.py", line 251, in __init__ 55 | (code, msg) = self.connect(host, port) 56 | File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/smtplib.py", line 311, in connect 57 | self.sock = self._get_socket(host, port, self.timeout) 58 | File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/smtplib.py", line 286, in _get_socket 59 | return socket.create_connection((host, port), timeout) 60 | File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 553, in create_connection 61 | for res in getaddrinfo(host, port, 0, SOCK_STREAM): 62 | gaierror: [Errno 8] nodename nor servname provided, or not known\ 63 | """) 64 | elif message_size == 'medium': 65 | message = ("Built-in functions, exceptions, and other objects.\n\nNoteworthy: " 66 | "None is the `nil' object;