├── LICENSE ├── README.md ├── setup.py └── sqlalchemy-s3sqlite ├── __init__.py └── dialect.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 cariaso 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqlalchemy-s3sqlite 2 | persist a sqlite database in s3, for use with aws lambda 3 | 4 | This module allows Flask-User to be used inside AWS Lambda. Normally the sqlite database would randomly disappear between invocations. This module instead makes the database be copied to and from s3 storage as needed. Performance will of course suffer, and having more than one concurrent user is either impossible, or a recipe for nasty surprises, but with it, it's possible to maintain a small site without the comparatively substantial cost of a t2.micro rds or similar. 5 | 6 | The idea and implementation borrows heavily from 7 | https://blog.zappa.io/posts/s3sqlite-a-serverless-relational-database 8 | but that implementation is very django dependent at the moment. Similarly this code still retains some of the rough aspects of that code, such as the repeated use of the hard coded '/tmp'. If it makes sense, I'd welcome having the two codes merge or consolidate in the future. 9 | 10 | Usage: based on quickstart_app.py from 11 | https://flask-user.readthedocs.io/en/latest/quickstart_app.html 12 | 13 | install the module 14 | ```pip install sqlalchemy-s3sqlite``` 15 | 16 | 17 | teach sqlite about s3sqlite 18 | ```python 19 | from sqlalchemy.dialects import registry 20 | registry.register("s3sqlite", "sqlalchemy-s3sqlite.dialect", "S3SQLiteDialect") 21 | ``` 22 | 23 | and change your SQLALCHEMY_DATABASE_URI 24 | ```python 25 | SQLALCHEMY_DATABASE_URI = os.getenv('DATABASE_URL', 's3sqlite:///quickstart_app.sqlite') 26 | ``` 27 | 28 | At the moment it remains dependent on an environment variable 29 | `S3SQLite_bucket` 30 | to know where to persist the sqlite database. For zappa users, this can be achieved with 31 | 32 | ```js 33 | "dev": { 34 | "environment_variables": { 35 | "S3SQLite_bucket": "mybucketname123" 36 | } 37 | } 38 | ```` 39 | Although I'm open to having a default of the zappa `s3_bucket` if others feel that's a worthwhile improvement. 40 | 41 | 42 | 43 | 44 | 45 | 46 | # Working Example 47 | 48 | ```python 49 | # This file contains an example Flask-User application. 50 | # To keep the example simple, we are applying some unusual techniques: 51 | # - Placing everything in one file 52 | # - Using class-based configuration (instead of file-based configuration) 53 | # - Using string-based templates (instead of file-based templates) 54 | 55 | from flask import Flask, render_template_string 56 | from flask_sqlalchemy import SQLAlchemy 57 | from flask_user import login_required, UserManager, UserMixin 58 | from flask_user import SQLAlchemyAdapter 59 | 60 | # Much better to set this via some other mechanism, but this keeps all 61 | # the settings in this one file 62 | import os 63 | os.environ['S3SQLite_bucket'] = 'MyBucketName' 64 | 65 | # this is the important change, it imports sqlalchemy-s3sqlite at runtime 66 | from sqlalchemy.dialects import registry 67 | registry.register("s3sqlite", "sqlalchemy-s3sqlite.dialect", "S3SQLiteDialect") 68 | 69 | 70 | # Class-based application configuration 71 | class ConfigClass(object): 72 | """ Flask application config """ 73 | 74 | # Flask settings 75 | SECRET_KEY = 'This is an INSECURE secret!! DO NOT use this in production!!' 76 | 77 | # Flask-SQLAlchemy settings 78 | SQLALCHEMY_DATABASE_URI = 'sqlite:///quickstart_app.sqlite' # File-based SQL database 79 | SQLALCHEMY_TRACK_MODIFICATIONS = False # Avoids SQLAlchemy warning 80 | 81 | # Flask-User settings 82 | USER_APP_NAME = "Flask-User QuickStart App" # Shown in and email templates and page footers 83 | USER_ENABLE_EMAIL = False # Disable email authentication 84 | USER_ENABLE_USERNAME = True # Enable username authentication 85 | USER_REQUIRE_RETYPE_PASSWORD = False # Simplify register form 86 | 87 | 88 | def create_app(): 89 | """ Flask application factory """ 90 | 91 | # Create Flask app load app.config 92 | app = Flask(__name__) 93 | app.config.from_object(__name__+'.ConfigClass') 94 | 95 | # Initialize Flask-SQLAlchemy 96 | db = SQLAlchemy(app) 97 | 98 | # Define the User data-model. 99 | # NB: Make sure to add flask_user UserMixin !!! 100 | class User(db.Model, UserMixin): 101 | __tablename__ = 'users' 102 | id = db.Column(db.Integer, primary_key=True) 103 | active = db.Column('is_active', db.Boolean(), nullable=False, server_default='1') 104 | 105 | # User authentication information 106 | username = db.Column(db.String(100), nullable=False, unique=True) 107 | password = db.Column(db.String(255), nullable=False, server_default='') 108 | email_confirmed_at = db.Column(db.DateTime()) 109 | 110 | # User information 111 | first_name = db.Column(db.String(100), nullable=False, server_default='') 112 | last_name = db.Column(db.String(100), nullable=False, server_default='') 113 | 114 | # Create all database tables 115 | db.create_all() 116 | 117 | # Setup Flask-User and specify the User data-model 118 | db_adapter = SQLAlchemyAdapter(db, User) # Register the User model 119 | user_manager = UserManager(db_adapter, app) # Initialize Flask-User 120 | 121 | # The Home page is accessible to anyone 122 | @app.route('/') 123 | def home_page(): 124 | # String-based templates 125 | return render_template_string(""" 126 | {% extends "flask_user_layout.html" %} 127 | {% block content %} 128 |

Home page

129 |

Register

130 |

Sign in

131 |

Home page (accessible to anyone)

132 |

Member page (login required)

133 |

Sign out

134 | {% endblock %} 135 | """) 136 | 137 | # The Members page is only accessible to authenticated users via the @login_required decorator 138 | @app.route('/members') 139 | @login_required # User must be authenticated 140 | def member_page(): 141 | # String-based templates 142 | return render_template_string(""" 143 | {% extends "flask_user_layout.html" %} 144 | {% block content %} 145 |

Members page

146 |

Register

147 |

Sign in

148 |

Home page (accessible to anyone)

149 |

Member page (login required)

150 |

Sign out

151 | {% endblock %} 152 | """) 153 | 154 | return app 155 | 156 | 157 | # Start development web server 158 | if __name__=='__main__': 159 | app = create_app() 160 | app.run(host='0.0.0.0', port=5000, debug=True) 161 | 162 | ``` 163 | 164 | 165 | # Warnings and future directions 166 | 167 | Consistent with the equivalent django code, it assumes databases which are explicitly stored below /tmp/ (or more accurately in a path containing '/tmp/' !) should not be persisted, so 168 | ```python 169 | SQLALCHEMY_DATABASE_URI = os.getenv('DATABASE_URL', 's3sqlite:////tmp/quickstart_app.sqlite') 170 | ``` 171 | would not be persisted. However this seems a bit silly since the s3sqlite dialect was explicitly stated. In time it may be worthwhile if this supports the other approaches shown at https://github.com/hkwi/sqlalchemy_gevent 172 | 173 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup(name='sqlalchemy-s3sqlite', 4 | version='0.1', 5 | description='persist a sqlite database in s3, for use with lambda', 6 | url='http://github.com/cariaso/sqlalchemy-s3sqlite', 7 | author='cariaso', 8 | author_email='cariaso@gmail.com', 9 | license='MIT', 10 | packages=['sqlalchemy-s3sqlite'], 11 | zip_safe=False) 12 | -------------------------------------------------------------------------------- /sqlalchemy-s3sqlite/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cariaso/sqlalchemy-s3sqlite/73b95c0120a5c8aa1a5836223ee2f7ac809d58d8/sqlalchemy-s3sqlite/__init__.py -------------------------------------------------------------------------------- /sqlalchemy-s3sqlite/dialect.py: -------------------------------------------------------------------------------- 1 | 2 | # based on 3 | # https://blog.zappa.io/posts/s3sqlite-a-serverless-relational-database 4 | # https://github.com/Miserlou/zappa-django-utils/blob/master/zappa_django_utils/db/backends/s3sqlite/base.py 5 | 6 | from io import BytesIO 7 | 8 | import boto3 9 | import botocore 10 | import hashlib 11 | import logging 12 | import os 13 | 14 | #log = logging.getLogger() 15 | #log.setLevel(logging.DEBUG) 16 | 17 | from sqlalchemy.dialects.sqlite.pysqlite import SQLiteDialect_pysqlite 18 | class S3SQLiteDialect(SQLiteDialect_pysqlite): 19 | 20 | 21 | def load_remote_db(self, dbname=None): 22 | """ 23 | Load remote S3 DB 24 | 25 | This code is still pretty ugly, but it seems to work, and remains fairly similar to 26 | # https://github.com/Miserlou/zappa-django-utils/blob/master/zappa_django_utils/db/backends/s3sqlite/base.py 27 | 28 | """ 29 | 30 | # user set a simple relative path in SQLALCHEMY_DATABASE_URI, but it was changed to an abspath 31 | self._local_dbname = os.path.relpath(dbname) 32 | self.db_hash = None 33 | 34 | bucketname = os.environ.get('S3SQLite_bucket') 35 | self._remote_dbname = self._local_dbname 36 | if '/tmp/' not in self._local_dbname: 37 | try: 38 | etag = '' 39 | if os.path.isfile('/tmp/' + self._local_dbname): 40 | m = hashlib.md5() 41 | with open('/tmp/' + self._local_dbname, 'rb') as f: 42 | m.update(f.read()) 43 | 44 | # In general the ETag is the md5 of the file, in some cases it's not, 45 | # and in that case we will just need to reload the file, I don't see any other way 46 | etag = m.hexdigest() 47 | 48 | signature_version = "s3v4" 49 | s3 = boto3.resource( 50 | 's3', 51 | config=botocore.client.Config(signature_version=signature_version), 52 | ) 53 | obj = s3.Object(bucketname, self._local_dbname) 54 | obj_bytes = obj.get(IfNoneMatch=etag)["Body"] # Will throw E on 304 or 404 55 | 56 | with open('/tmp/' + self._local_dbname, 'wb') as f: 57 | f.write(obj_bytes.read()) 58 | 59 | m = hashlib.md5() 60 | with open('/tmp/' + self._local_dbname, 'rb') as f: 61 | m.update(f.read()) 62 | 63 | self.db_hash = m.hexdigest() 64 | 65 | logging.debug("Loaded remote DB!") 66 | except botocore.exceptions.ClientError as e: 67 | error_code = e.response['Error']['Code'] 68 | if error_code == '304': 69 | logging.debug("ETag matches md5 of local copy, using local copy of DB!") 70 | self.db_hash = etag 71 | else: 72 | logging.debug("Couldn't load remote DB object.") 73 | except Exception as e: 74 | # Weird one 75 | logging.debug(e) 76 | 77 | 78 | if '/tmp/' not in self._local_dbname: 79 | self._local_dbname = '/tmp/' + self._local_dbname 80 | 81 | return self._local_dbname 82 | 83 | def close(self, *args, **kwargs): 84 | """ 85 | Engine closed, copy file to DB if it has changed 86 | """ 87 | 88 | bucketname = os.environ.get('S3SQLite_bucket') 89 | try: 90 | with open(self._local_dbname, 'rb') as f: 91 | fb = f.read() 92 | 93 | m = hashlib.md5() 94 | m.update(fb) 95 | if self.db_hash == m.hexdigest(): 96 | logging.debug("Database unchanged, not saving to remote DB!") 97 | return 98 | 99 | bytesIO = BytesIO() 100 | bytesIO.write(fb) 101 | bytesIO.seek(0) 102 | 103 | signature_version = "s3v4" 104 | s3 = boto3.resource( 105 | 's3', 106 | config=botocore.client.Config(signature_version=signature_version), 107 | ) 108 | 109 | s3_object = s3.Object(bucketname, self._remote_dbname) 110 | result = s3_object.put(Body=bytesIO) 111 | logging.debug("Saved to remote DB!") 112 | except Exception as e: 113 | logging.debug(e) 114 | 115 | 116 | 117 | 118 | def __init__(self, *args, **kw): 119 | #print("S3SQLiteDialect.__init__(args=%s, kw=%s)" % (args, kw)) 120 | super(S3SQLiteDialect, self).__init__(*args, **kw) 121 | def connect(self, *args, **kw): 122 | #print("S3SQLiteDialect.connect(args=%s, kw=%s)" % (args, kw)) 123 | localname = self.load_remote_db(dbname=args[0]) 124 | return super(S3SQLiteDialect, self).connect(localname, **kw) 125 | def do_close(self, *args, **kw): 126 | #print("S3SQLiteDialect.do_close(args=%s, kw=%s)" % (args, kw)) 127 | out = super(S3SQLiteDialect, self).do_close(*args, **kw) 128 | self.close() 129 | return out 130 | 131 | 132 | --------------------------------------------------------------------------------