├── .github └── workflows │ └── docker-image.yml ├── Dockerfile ├── README.md ├── docker-compose.yml ├── examples └── docker-compose.yml ├── my.cnf ├── scripts ├── create_branch.sh ├── delete_branch.sh ├── entrypoint.sh └── list_branches.sh └── web ├── main.py └── requirements.txt /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image CI 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: 7 | branches: ["main"] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v3 16 | - name: Login to DockerHub 17 | uses: docker/login-action@v2 18 | with: 19 | username: ${{ secrets.DOCKERHUB_USERNAME }} 20 | password: ${{ secrets.DOCKERHUB_TOKEN }} 21 | - name: Build and push 22 | uses: docker/build-push-action@v3 23 | with: 24 | context: . 25 | push: true 26 | tags: mliezun/branchable-mysql:latest 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | ARG DEBIAN_FRONTEND=noninteractive 4 | 5 | WORKDIR /app 6 | 7 | RUN apt-get update -yyqq && \ 8 | apt-get install -yyqq mysql-server python3 python3-pip fuse-overlayfs curl strace && \ 9 | mkdir -p layers/base/conf && \ 10 | mkdir -p layers/base/data && \ 11 | mkdir -p layers/base/logs && \ 12 | mkdir -p layers/base/var/lib/mysql-files && \ 13 | chown -R mysql:mysql layers/base 14 | 15 | COPY my.cnf layers/base/conf/ 16 | RUN mysqld \ 17 | --initialize-insecure \ 18 | --datadir=/app/layers/base/data/ \ 19 | --pid-file=/app/layers/base/var/mysqld.pid \ 20 | --socket=/app/layers/base/var/mysqld.sock \ 21 | --secure-file-priv=/app/layers/base/var/lib/mysql-files \ 22 | --port=33061 \ 23 | --log-error=/app/layers/base/logs/error.log \ 24 | --log-bin=/app/layers/base/var/mysql-bin.log \ 25 | --slow-query-log-file=/app/layers/base/logs/slow_query.log \ 26 | --general-log-file=/app/layers/base/logs/query.log \ 27 | --user=mysql \ 28 | --bind-address=127.0.0.1 || true 29 | 30 | COPY scripts scripts 31 | 32 | COPY web/requirements.txt web/requirements.txt 33 | RUN pip install -r web/requirements.txt 34 | 35 | COPY web/main.py web/main.py 36 | 37 | 38 | ENTRYPOINT [ "./scripts/entrypoint.sh" ] 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Branchable MySQL 2 | 3 | Create branches on your MySQL databases to have multiple dev environments. 4 | 5 | When teams start to grow, having a single dev environment becomes an issue. People start stepping on each others toes. 6 | A common problem is that two people want to apply incompatible migrations on the database. That problem is impossible 7 | to fix if folks are working on parallel branches. 8 | If we can have a database for each branch of a project, that will remove much of the pain of having multiple devs applying 9 | changes to the db. 10 | 11 | Related [blogpost](https://mliezun.github.io/2022/09/20/branchable-mysql.html). 12 | 13 | ## Usage 14 | 15 | Create a file like the one located in `examples/docker-compose.yml` 16 | 17 | ```yaml 18 | version: "3" 19 | 20 | services: 21 | mysql: 22 | image: mliezun/branchable-mysql 23 | platform: linux/amd64 24 | privileged: true 25 | restart: always 26 | volumes: 27 | - appdata:/app/ 28 | 29 | volumes: 30 | appdata: 31 | ``` 32 | 33 | Execute `docker compose up` to initialize the container. 34 | 35 | Then you can access handful scripts inside the container. 36 | 37 | 38 | ### Connect to the `base` database inside the container 39 | 40 | ```shell 41 | $ docker compose exec mysql mysql -uroot -h127.0.0.1 --skip-password -P33061 42 | mysql: [Warning] Using a password on the command line interface can be insecure. 43 | Welcome to the MySQL monitor. Commands end with ; or \g. 44 | Your MySQL connection id is 8 45 | Server version: 8.0.30-0ubuntu0.22.04.1 (Ubuntu) 46 | 47 | Copyright (c) 2000, 2022, Oracle and/or its affiliates. 48 | 49 | Oracle is a registered trademark of Oracle Corporation and/or its 50 | affiliates. Other names may be trademarks of their respective 51 | owners. 52 | 53 | Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. 54 | 55 | mysql> 56 | ``` 57 | 58 | 59 | ### Create branch from database 60 | 61 | ```shell 62 | $ docker compose exec mysql /app/scripts/create_branch.sh base feature/abc 63 | {"branch_name":"feature/abc", "base_branch":"base", "port":33062} 64 | ``` 65 | 66 | To be able to use the new branch connect using the new port: 67 | 68 | ```shell 69 | $ docker compose exec mysql mysql -uroot -h127.0.0.1 --skip-password -P33062 70 | ``` 71 | 72 | ### List branches 73 | 74 | ```shell 75 | $ docker compose exec mysql /app/scripts/list_branches.sh 76 | [{"branch_name":"base"}, {"branch_name":"feature/abc"}] 77 | ``` 78 | 79 | ### Delete branch from database (will drop all the data) 80 | 81 | ```shell 82 | $ docker compose exec mysql /app/scripts/delete_branch.sh feature/abc 83 | {"branch_name":"feature/abc"} 84 | ``` 85 | 86 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | api: 5 | platform: linux/amd64 6 | privileged: true 7 | build: 8 | context: . 9 | dockerfile: ./Dockerfile 10 | restart: always 11 | volumes: 12 | - ./web:/app/web 13 | -------------------------------------------------------------------------------- /examples/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | mysql: 5 | image: mliezun/branchable-mysql 6 | platform: linux/amd64 7 | privileged: true 8 | restart: always 9 | volumes: 10 | - appdata:/app/ 11 | 12 | volumes: 13 | appdata: 14 | -------------------------------------------------------------------------------- /my.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | -------------------------------------------------------------------------------- /scripts/create_branch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | BASE_BRANCH=$1 6 | NEW_BRANCH=$2 7 | 8 | function print_usage() { 9 | echo "create_branch.sh BASE_BRANCH NEW_BRANCH"; 10 | } 11 | 12 | if [ -z ${BASE_BRANCH+x} ]; then 13 | print_usage; 14 | echo "BASE_BRANCH is required"; 15 | exit 1; 16 | fi 17 | 18 | if [ -z ${NEW_BRANCH+x} ]; then 19 | print_usage; 20 | echo "NEW_BRANCH is required"; 21 | exit 1; 22 | fi 23 | 24 | 25 | curl -X POST http://127.0.0.1:8000/create-branch \ 26 | -H 'Content-Type: application/json' \ 27 | -d "{\"base_branch\": \"$BASE_BRANCH\", \"branch_name\": \"$NEW_BRANCH\"}" 28 | -------------------------------------------------------------------------------- /scripts/delete_branch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | BRANCH_NAME=$1 6 | 7 | function print_usage() { 8 | echo "delete_branch.sh BRANCH_NAME"; 9 | } 10 | 11 | if [ -z ${BRANCH_NAME+x} ]; then 12 | print_usage; 13 | echo "BRANCH_NAME is required"; 14 | exit 1; 15 | fi 16 | 17 | curl -X DELETE http://127.0.0.1:8000/delete-branch \ 18 | -H 'Content-Type: application/json' \ 19 | -d "{\"branch_name\": \"$BRANCH_NAME\"}" 20 | -------------------------------------------------------------------------------- /scripts/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd web && \ 4 | uvicorn main:app 5 | -------------------------------------------------------------------------------- /scripts/list_branches.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | curl -X GET http://127.0.0.1:8000/list-branches 6 | -------------------------------------------------------------------------------- /web/main.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import uuid 4 | from typing import List 5 | 6 | import sh 7 | from fastapi import FastAPI 8 | from fastapi.responses import JSONResponse 9 | from peewee import (CharField, DateTimeField, ForeignKeyField, IntegerField, 10 | Model, SqliteDatabase, UUIDField) 11 | from pydantic import BaseModel 12 | 13 | logging.basicConfig(level=logging.INFO) 14 | 15 | app = FastAPI() 16 | sqlite_db = SqliteDatabase("app.db") 17 | 18 | 19 | class DBBaseModel(Model): 20 | """A base model that will use our Sqlite database""" 21 | id = UUIDField(primary_key=True, default=uuid.uuid4) 22 | created = DateTimeField(default=datetime.datetime.now) 23 | 24 | class Meta: 25 | database = sqlite_db 26 | 27 | 28 | class Layer(DBBaseModel): 29 | bottom_layer = ForeignKeyField("self", null=True) 30 | 31 | 32 | class Port(DBBaseModel): 33 | n = IntegerField(unique=True) 34 | 35 | class Branch(DBBaseModel): 36 | branch_name = CharField(unique=True) 37 | port = ForeignKeyField(Port) 38 | layer = ForeignKeyField(Layer, default=Layer.create) 39 | 40 | class CreateBranch(BaseModel): 41 | branch_name: str 42 | base_branch: str 43 | port: int | None 44 | 45 | class BranchModel(BaseModel): 46 | branch_name: str 47 | 48 | class APIMessage(BaseModel): 49 | message: str 50 | 51 | 52 | BASE_PORT = 33061 53 | def get_port(): 54 | for p in Port.select().iterator(): 55 | if not Branch.filter(port=p).exists(): 56 | return p 57 | raise Port.DoesNotExist 58 | 59 | 60 | def start_mysqld(layer: str, port: int): 61 | return sh.mysqld( 62 | f"--defaults-file=/app/mysql/{layer}/conf/my.cnf", 63 | f"--datadir=/app/mysql/{layer}/data/", 64 | f"--pid-file=/app/mysql/{layer}/var/mysqld.pid", 65 | f"--socket=/app/mysql/{layer}/var/mysqld.sock", 66 | f"--secure-file-priv=/app/mysql/{layer}/var/lib/mysql-files", 67 | f"--port={port}", 68 | f"--log-error=/app/mysql/{layer}/logs/error.log", 69 | f"--log-bin=/app/mysql/{layer}/var/mysql-bin.log", 70 | f"--slow-query-log-file=/app/mysql/{layer}/logs/slow_query.log", 71 | f"--general-log-file=/app/mysql/{layer}/logs/query.log", 72 | f"--user=mysql", 73 | f"--bind-address=127.0.0.1", 74 | _bg=True, 75 | _out=logging.info, 76 | _err=logging.warn, 77 | ) 78 | 79 | 80 | def mount_layer(bottom_layers: List[str], new_layer: str): 81 | sh.mkdir( 82 | "-p", 83 | f"/app/layers/{new_layer}", 84 | _out=logging.info, 85 | _err=logging.warn, 86 | ) 87 | sh.mkdir( 88 | "-p", 89 | f"/app/tmp/{new_layer}", 90 | _out=logging.info, 91 | _err=logging.warn, 92 | ) 93 | sh.mkdir( 94 | "-p", 95 | f"/app/mysql/{new_layer}", 96 | _out=logging.info, 97 | _err=logging.warn, 98 | ) 99 | lowerdir = ':'.join([f"/app/layers/{bottom_layer}" for bottom_layer in bottom_layers] + ["/app/layers/base"]) 100 | sh.fuse_overlayfs( 101 | "-o", 102 | f"lowerdir={lowerdir},upperdir=/app/layers/{new_layer},workdir=/app/tmp/{new_layer}", 103 | "overlay", 104 | f"/app/mysql/{new_layer}", 105 | _out=logging.info, 106 | _err=logging.warn, 107 | ) 108 | 109 | def umount_layer(layer): 110 | sh.umount( 111 | f"/app/mysql/{layer}", 112 | _out=logging.info, 113 | _err=logging.warn, 114 | ) 115 | 116 | 117 | processes = {} 118 | 119 | 120 | @app.post("/create-branch", response_model=CreateBranch, responses={404: {"model": APIMessage}}) 121 | def create_branch(branch: CreateBranch): 122 | if not Branch.filter(branch_name=branch.base_branch).exists(): 123 | return JSONResponse(status_code=404, content={"message": "Base branch not found"}) 124 | 125 | if Branch.filter(branch_name=branch.branch_name).exists(): 126 | return JSONResponse(status_code=400, content={"message": "Branch already exists"}) 127 | 128 | 129 | base_branch = Branch.get(branch_name=branch.base_branch) 130 | bottom_layer = str(base_branch.layer.id) 131 | base_proc = processes[branch.base_branch] 132 | del processes[branch.base_branch] 133 | base_proc.terminate() 134 | base_proc.wait() 135 | logging.info("Umounted layer") 136 | 137 | base_new_layer = Layer.create(bottom_layer=bottom_layer) 138 | layer = str(base_new_layer.id) 139 | previous_layers = [] 140 | l = base_new_layer.bottom_layer 141 | while l: 142 | previous_layers.append(str(l.id)) 143 | l = l.bottom_layer 144 | mount_layer(previous_layers, layer) 145 | processes[branch.base_branch] = start_mysqld(layer, base_branch.port.n) 146 | base_branch.layer = layer 147 | base_branch.save() 148 | 149 | new_layer = Layer.create(bottom_layer=bottom_layer) 150 | 151 | port = get_port() 152 | new_branch = Branch.create(branch_name=branch.branch_name, layer=new_layer, port=port) 153 | 154 | layer = str(new_branch.layer.id) 155 | mount_layer(previous_layers, layer) 156 | processes[branch.branch_name] = start_mysqld(layer, port.n) 157 | 158 | branch.port = port.n 159 | return branch 160 | 161 | @app.delete("/delete-branch", response_model=BranchModel, responses={404: {"model": APIMessage}}) 162 | def delete_branch(branch: BranchModel): 163 | if not Branch.filter(branch_name=branch.branch_name).exists(): 164 | return JSONResponse(status_code=404, content={"message": "Branch not found"}) 165 | 166 | if branch.branch_name == "base": 167 | return JSONResponse(status_code=400, content={"message": "Cannot delete base branch"}) 168 | 169 | Branch.get(Branch.branch_name == branch.branch_name).delete_instance() 170 | 171 | proc = processes[branch.branch_name] 172 | del processes[branch.branch_name] 173 | proc.terminate() 174 | proc.wait() 175 | 176 | return branch 177 | 178 | 179 | @app.get("/list-branches", response_model=List[BranchModel], responses={404: {"model": APIMessage}}) 180 | def delete_branch(): 181 | return [b for b in Branch.select().dicts()] 182 | 183 | 184 | def startup(): 185 | Port.create_table() 186 | Layer.create_table() 187 | Branch.create_table() 188 | 189 | for i in range(100): 190 | Port.get_or_create(n=BASE_PORT+i) 191 | 192 | port = BASE_PORT 193 | base_branch, _ = Branch.get_or_create(branch_name="base", port=Port.get(n=BASE_PORT)) 194 | layer = str(base_branch.layer.id) 195 | mount_layer([], layer) 196 | processes["base"] = start_mysqld(layer, port) 197 | 198 | for b in Branch.select().iterator(): 199 | if b.branch_name == "base": 200 | continue 201 | previous_layers = [] 202 | l = b.layer 203 | while l: 204 | previous_layers.append(str(l.id)) 205 | l = l.bottom_layer 206 | mount_layer(previous_layers[1:], previous_layers[0]) 207 | processes[b.branch_name] = start_mysqld(previous_layers[0], b.port.n) 208 | 209 | startup() 210 | -------------------------------------------------------------------------------- /web/requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | uvicorn 3 | sh 4 | peewee 5 | pydantic --------------------------------------------------------------------------------