├── LICENSE ├── README.md ├── composer.json ├── phpstan.neon └── src ├── Bref ├── ConnectionTracker.php ├── EventSubscriber.php └── init.php ├── DbSynchronizer.php ├── Laravel ├── SQLiteS3ServiceProvider.php └── SqliteS3Connector.php ├── PDOSQLiteS3.php └── SQLiteS3.php /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 Matthieu Napoli 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Serverless dev database: SQLite backed by S3 2 | 3 | ## Why? 4 | 5 | A "serverless" SQL database: 6 | 7 | - For development and testing purposes 8 | - Costs $0 9 | - As simple to use as possible 10 | - Ideal for [serverless environments](https://bref.sh/) like AWS Lambda 11 | 12 | **Not for production use-cases**. It does not handle concurrent updates (in which case some data might be lost) and performances are not production-grade. 13 | 14 | ## How? 15 | 16 | The SQLite database (a file) is stored on S3. The PHP class will transparently download the file locally on every request, and upload it back at the end. 17 | 18 | This has two obvious implications: 19 | 20 | 1. If two concurrent requests download the database file, update it (separately), and upload it back, then the last to upload the modified file will overwrite the changes of the other request. 21 | 2. There is extra latency added to the request (and it wouldn't work well with huge databases). 22 | 23 | That is why this solution is best for testing scenarios (e.g. testing a fully deployed application, where there is one test running at a time). It could also work for development environments with only one active user at a time, where an extra 50ms-100ms per request is acceptable. 24 | 25 | ## Setup 26 | 27 | You will need an AWS S3 bucket (where the database will be stored). The S3 bucket must exist, but **the SQLite database file will automatically be created** if it doesn't. 28 | 29 | Install the package with Composer: 30 | 31 | ```sh 32 | composer require mnapoli/sqlite-s3 33 | ``` 34 | 35 | ## Usage 36 | 37 | ### With Laravel 38 | 39 | Update `.env` (or set environment variables) to set: 40 | 41 | - `DB_CONNECTION=sqlite` 42 | - `DB_DATABASE='s3://the-s3-bucket-name/a-file-name.sqlite'` 43 | 44 | The `DB_DATABASE` usually contains a file name, but here it will contain a S3 URL. That URL will be automatically detected to retrieve the database from S3. 45 | 46 | The database will be uploaded to S3 on every request. When running on AWS Lambda with Bref, it will be uploaded/synced on every AWS Lambda invocation too. 47 | 48 | Outside of Lambda (for example in test code), call `DB::purge();` to force the database to be synced to S3. 49 | 50 | ### Generic PHP application 51 | 52 | Instead of: 53 | 54 | ```php 55 | $db = new PDO('sqlite:test-db.sqlite'); 56 | $db->exec('SELECT * FROM my-table'); 57 | ``` 58 | 59 | Use: 60 | 61 | ```php 62 | $db = new PDOSQLiteS3('the-s3-bucket-name', 'a-file-name.sqlite'); 63 | $db->exec('SELECT * FROM my-table'); 64 | ``` 65 | 66 | The database will be uploaded back to S3 when the `$db` instance is destroyed (i.e. when the PDO connection is closed). 67 | 68 | ### Configuration 69 | 70 | If needed, set the AWS region: 71 | 72 | ```php 73 | $db = new PDOSQLiteS3('the-s3-bucket-name', 'a-file-name.sqlite', [ 74 | 'region' => 'us-east-1', 75 | ]); 76 | ``` 77 | 78 | The AWS credentials will automatically be picked up by the AWS SDK. The [Async-AWS](https://async-aws.com/) library is used under the hood, check out [its documentation](https://async-aws.com/authentication/). 79 | 80 | ### Without PDO 81 | 82 | If you are using the [`SQLite3` class](https://www.php.net/manual/en/class.sqlite3.php) directly, replace it with the `SQLiteS3` class: 83 | 84 | ```php 85 | $db = new SQLiteS3('the-s3-bucket-name', 'a-file-name.sqlite'); 86 | ``` 87 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mnapoli/sqlite-s3", 3 | "description": "", 4 | "keywords": ["sqlite", "s3", "aws", "serverless"], 5 | "license": "MIT", 6 | "type": "library", 7 | "autoload": { 8 | "psr-4": { 9 | "SQLiteS3\\": "src/" 10 | }, 11 | "files": [ 12 | "src/Bref/init.php" 13 | ] 14 | }, 15 | "autoload-dev": { 16 | "psr-4": { 17 | "SQLiteS3\\Test\\": "tests/" 18 | } 19 | }, 20 | "require": { 21 | "php": ">=8.1", 22 | "ext-pdo": "*", 23 | "ext-sqlite3": "*", 24 | "async-aws/simple-s3": "^1|^2", 25 | "bref/bref": "^2.1.10", 26 | "bref/logger": "^1.0" 27 | }, 28 | "require-dev": { 29 | "phpunit/phpunit": "^10.5", 30 | "mnapoli/hard-mode": "^0.3", 31 | "phpstan/phpstan": "^1", 32 | "laravel/framework": "^10.39" 33 | }, 34 | "config": { 35 | "allow-plugins": { 36 | "dealerdirect/phpcodesniffer-composer-installer": true, 37 | "pestphp/pest-plugin": false 38 | } 39 | }, 40 | "extra": { 41 | "laravel": { 42 | "providers": [ 43 | "SQLiteS3\\Laravel\\SQLiteS3ServiceProvider" 44 | ] 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /phpstan.neon: -------------------------------------------------------------------------------- 1 | parameters: 2 | level: 6 3 | paths: 4 | - src 5 | - tests 6 | -------------------------------------------------------------------------------- /src/Bref/ConnectionTracker.php: -------------------------------------------------------------------------------- 1 | */ 15 | private static WeakMap $connections; 16 | 17 | public static function trackConnection(PDOSQLiteS3 $connection): void 18 | { 19 | if (! isset(self::$connections)) { 20 | // @phpstan-ignore-next-line 21 | self::$connections = new WeakMap; 22 | } 23 | 24 | self::$connections[$connection] = true; 25 | } 26 | 27 | public static function closeAll(): void 28 | { 29 | if (isset(self::$connections)) { 30 | foreach (self::$connections as $connection => $_) { 31 | $connection->close(); 32 | } 33 | } 34 | 35 | // @phpstan-ignore-next-line 36 | self::$connections = new WeakMap; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/Bref/EventSubscriber.php: -------------------------------------------------------------------------------- 1 | subscribe(new EventSubscriber); 8 | -------------------------------------------------------------------------------- /src/DbSynchronizer.php: -------------------------------------------------------------------------------- 1 | $s3ClientConfig 24 | */ 25 | public function __construct( 26 | private readonly string $bucket, 27 | private readonly string $key, 28 | array|Configuration $s3ClientConfig = [], 29 | ) { 30 | $this->logger = new StderrLogger(LogLevel::INFO); 31 | $this->s3 = new SimpleS3Client($s3ClientConfig); 32 | } 33 | 34 | /** 35 | * @return string The path to the DB file name 36 | */ 37 | public function open(): string 38 | { 39 | $this->logger->info('Downloading and opening the SQLite database'); 40 | 41 | try { 42 | $contentAsResource = $this->s3->download($this->bucket, $this->key)->getContentAsResource(); 43 | } catch (NoSuchKeyException) { 44 | // The file does not exist yet, create an empty one 45 | $contentAsResource = fopen('php://memory', 'rb'); 46 | } 47 | 48 | $this->dbFileName = tempnam(sys_get_temp_dir(), 'db.sqlite'); 49 | if ($this->dbFileName === false) { 50 | throw new RuntimeException('Could not create temporary file'); 51 | } 52 | $fileResource = fopen($this->dbFileName, 'wb'); 53 | 54 | $success = stream_copy_to_stream($contentAsResource, $fileResource); 55 | if ($success === false) { 56 | throw new RuntimeException('Could not dump S3 file to temporary file'); 57 | } 58 | if (! fclose($fileResource)) { 59 | throw new RuntimeException('Could not close temporary file'); 60 | } 61 | 62 | $this->dbFileHash = md5_file($this->dbFileName); 63 | if ($this->dbFileHash === false) { 64 | throw new RuntimeException('Could not calculate MD5 hash'); 65 | } 66 | 67 | return $this->dbFileName; 68 | } 69 | 70 | public function close(): void 71 | { 72 | if (! $this->dbFileName) { 73 | return; 74 | } 75 | 76 | $fileChanged = $this->dbFileHash !== md5_file($this->dbFileName); 77 | 78 | $this->logger->info('Closing' . ($fileChanged ? ' and uploading' : '') . ' the SQLite database'); 79 | 80 | if ($fileChanged) { 81 | // Upload back to S3 82 | $contentAsResource = fopen($this->dbFileName, 'rb'); 83 | $this->s3->upload($this->bucket, $this->key, $contentAsResource); 84 | fclose($contentAsResource); 85 | } 86 | 87 | unlink($this->dbFileName); 88 | 89 | $this->dbFileName = null; 90 | } 91 | 92 | public function isOpened(): bool 93 | { 94 | return $this->dbFileName !== null; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/Laravel/SQLiteS3ServiceProvider.php: -------------------------------------------------------------------------------- 1 | app->bind('db.connector.sqlite', SqliteS3Connector::class); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Laravel/SqliteS3Connector.php: -------------------------------------------------------------------------------- 1 | $config 14 | * 15 | * @see \Illuminate\Database\Connectors\SQLiteConnector::connect() 16 | */ 17 | public function connect(array $config): PDO 18 | { 19 | $options = $this->getOptions($config); 20 | 21 | if (str_starts_with($config['database'] ?? '', 's3://')) { 22 | return $this->createConnection("sqlite-s3:{$config['database']}", $config, $options); 23 | } 24 | 25 | return parent::connect($config); 26 | } 27 | 28 | /** 29 | * @param array $options 30 | */ 31 | protected function createPdoConnection($dsn, $username, $password, $options): PDO 32 | { 33 | if (str_starts_with($dsn, 'sqlite-s3:')) { 34 | $success = preg_match('/s3:\/\/([^\/]+)\/(.*)/', $dsn, $matches); 35 | if (! $success) { 36 | throw new RuntimeException('Could not parse DSN: ' . $dsn); 37 | } 38 | [$_, $bucket, $key] = $matches; 39 | return new PDOSQLiteS3($bucket, $key); 40 | } 41 | 42 | return parent::createPdoConnection($dsn, $username, $password, $options); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/PDOSQLiteS3.php: -------------------------------------------------------------------------------- 1 | $s3ClientConfig 17 | */ 18 | public function __construct(string $bucket, string $key, array|Configuration $s3ClientConfig = []) 19 | { 20 | $this->dbSynchronizer = new DbSynchronizer($bucket, $key, $s3ClientConfig); 21 | 22 | $dbFileName = $this->dbSynchronizer->open(); 23 | 24 | parent::__construct('sqlite:' . $dbFileName); 25 | 26 | ConnectionTracker::trackConnection($this); 27 | } 28 | 29 | public function __destruct() 30 | { 31 | $this->close(); 32 | } 33 | 34 | public function close(): void 35 | { 36 | $this->dbSynchronizer->close(); 37 | } 38 | 39 | private function ensureIsNotClosed(): void 40 | { 41 | if (! $this->dbSynchronizer->isOpened()) { 42 | // The following error message is constructed so that Laravel detects this as a closed connection 43 | // and automatically reconnects to the database. 44 | // This is done via the magic "Lost connection" string. 45 | throw new RuntimeException('The SQLite database has been closed and uploaded to S3. You need to re-open the PDO connection. Lost connection'); 46 | } 47 | } 48 | 49 | public function prepare(...$params): PDOStatement|false 50 | { 51 | $this->ensureIsNotClosed(); 52 | 53 | return parent::prepare(...$params); 54 | } 55 | 56 | public function beginTransaction(): bool 57 | { 58 | $this->ensureIsNotClosed(); 59 | 60 | return parent::beginTransaction(); 61 | } 62 | 63 | public function commit(): bool 64 | { 65 | $this->ensureIsNotClosed(); 66 | 67 | return parent::commit(); 68 | } 69 | 70 | public function rollBack(): bool 71 | { 72 | $this->ensureIsNotClosed(); 73 | 74 | return parent::rollBack(); 75 | } 76 | 77 | public function exec(...$params): int|false 78 | { 79 | $this->ensureIsNotClosed(); 80 | 81 | return parent::exec(...$params); 82 | } 83 | 84 | public function query(mixed ...$params) 85 | { 86 | $this->ensureIsNotClosed(); 87 | 88 | return parent::query(...$params); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/SQLiteS3.php: -------------------------------------------------------------------------------- 1 | $s3ClientConfig 15 | */ 16 | public function __construct(string $bucket, string $key, array|Configuration $s3ClientConfig = []) 17 | { 18 | $this->dbSynchronizer = new DbSynchronizer($bucket, $key, $s3ClientConfig); 19 | 20 | $dbFileName = $this->dbSynchronizer->open(); 21 | 22 | parent::__construct($dbFileName); 23 | } 24 | 25 | public function close(): bool 26 | { 27 | $success = parent::close(); 28 | if ($success === false) { 29 | throw new RuntimeException('Could not close SQLite3 database'); 30 | } 31 | 32 | $this->dbSynchronizer->close(); 33 | 34 | return $success; 35 | } 36 | 37 | public function __destruct() 38 | { 39 | $this->close(); 40 | } 41 | } 42 | --------------------------------------------------------------------------------