├── bin ├── clean ├── build ├── start ├── debug ├── restore ├── push └── restore-backup-start ├── .gitignore ├── image ├── transactor.properties ├── start.sh ├── create-db.bsh └── Dockerfile ├── repl.clj └── README.md /bin/clean: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | rm -rf storage 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | storage 2 | mbrainz.tar 3 | mbrainz-1968-1973 4 | backup 5 | tmp/ 6 | -------------------------------------------------------------------------------- /bin/build: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | docker build --platform linux/amd64,linux/arm64 --tag filipesilva/datomic-pro-sqlite:latest image/ 4 | -------------------------------------------------------------------------------- /bin/start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | docker run -p 4334:4334 -v ./storage:/usr/storage --name datomic-pro-sqlite-demo --rm filipesilva/datomic-pro-sqlite:latest 4 | -------------------------------------------------------------------------------- /bin/debug: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | docker run -p 4334:4334 -v ./storage:/usr/storage --name datomic-pro-sqlite-demo --rm -it filipesilva/datomic-pro-sqlite:latest /bin/bash 4 | -------------------------------------------------------------------------------- /bin/restore: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | docker exec datomic-pro-sqlite-demo /usr/datomic-pro/bin/datomic restore-db file:///usr/restore/ datomic:sql://mbrainz-1968-1973?jdbc:sqlite:/usr/storage/sqlite.db 4 | -------------------------------------------------------------------------------- /image/transactor.properties: -------------------------------------------------------------------------------- 1 | port=4334 2 | host=0.0.0.0 3 | 4 | storage-access=remote 5 | protocol=sql 6 | sql-driver-class=org.sqlite.JDBC 7 | sql-url=jdbc:sqlite:/usr/storage/sqlite.db 8 | 9 | # See https://docs.datomic.com/on-prem/capacity.html 10 | memory-index-threshold=32m 11 | memory-index-max=256m 12 | object-cache-max=128m 13 | -------------------------------------------------------------------------------- /bin/push: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # SET THIS 4 | # use datomic version, and don't commit the change 5 | tag="" 6 | 7 | # Check if the variable is empty 8 | if [ -z "$tag" ]; then 9 | echo "ERROR: tag empty, set it before pushing!" 10 | exit 1 11 | fi 12 | 13 | docker tag filipesilva/datomic-pro-sqlite:latest filipesilva/datomic-pro-sqlite:$tag 14 | docker push filipesilva/datomic-pro-sqlite:$tag 15 | docker push filipesilva/datomic-pro-sqlite:latest 16 | -------------------------------------------------------------------------------- /bin/restore-backup-start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # get mbrainz backup if we don't have it yet 4 | if [ ! -d "mbrainz-1968-1973" ]; then 5 | curl https://s3.amazonaws.com/mbrainz/datomic-mbrainz-1968-1973-backup-2017-07-20.tar -o mbrainz.tar 6 | tar -xvf mbrainz.tar 7 | fi 8 | 9 | # mount mbrainz 10 | docker run -p 4334:4334 -v ./storage:/usr/storage -v ./mbrainz-1968-1973:/usr/restore -v ./backup:/usr/backup --name datomic-pro-sqlite-demo --rm filipesilva/datomic-pro-sqlite:latest 11 | -------------------------------------------------------------------------------- /repl.clj: -------------------------------------------------------------------------------- 1 | (add-libs {'com.datomic/peer {:mvn/version "1.0.7394"} 2 | 'org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}}) 3 | (require '[datomic.api :as d]) 4 | 5 | (def db-uri "datomic:sql://app?jdbc:sqlite:./storage/sqlite.db") 6 | (def conn (d/connect db-uri)) 7 | 8 | (d/transact conn [{:db/ident :foo}]) 9 | (d/pull (d/db conn) '[*] :foo) 10 | ;; => #:db{:id 17592186045417, :ident :foo} 11 | 12 | (def test-db-uri (str "datomic:mem://test-" (random-uuid))) 13 | (d/create-database test-db-uri) 14 | (def test-conn (d/connect test-db-uri)) 15 | (d/pull (d/db test-conn) '[*] :foo) 16 | ;; => #:db{:id nil} 17 | -------------------------------------------------------------------------------- /image/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -e 3 | 4 | # Create sqlite db if it doesn't exist 5 | if [ ! -f /usr/storage/sqlite.db ]; then 6 | echo "Creating sqlite database at /usr/storage/sqlite.db" 7 | sqlite3 /usr/storage/sqlite.db " 8 | -- same as Rails 8.0 9 | PRAGMA foreign_keys = ON; 10 | PRAGMA journal_mode = WAL; 11 | PRAGMA synchronous = NORMAL; 12 | PRAGMA mmap_size = 134217728; -- 128 megabytes 13 | PRAGMA journal_size_limit = 67108864; -- 64 megabytes 14 | PRAGMA cache_size = 2000; 15 | 16 | -- datomic schema 17 | CREATE TABLE IF NOT EXISTS datomic_kvs ( 18 | id TEXT NOT NULL, 19 | rev INTEGER, 20 | map TEXT, 21 | val BYTEA, 22 | CONSTRAINT pk_id PRIMARY KEY (id) 23 | ); 24 | " > /dev/null 25 | fi 26 | 27 | # Run db creation script in background 28 | /usr/datomic-pro/bin/shell /usr/create-db.bsh & 29 | 30 | # Run datomic transactor 31 | exec /usr/datomic-pro/bin/transactor /usr/datomic-pro/config/transactor.properties 32 | -------------------------------------------------------------------------------- /image/create-db.bsh: -------------------------------------------------------------------------------- 1 | db_name = System.getenv("DATOMIC_DB"); 2 | if (db_name == null || db_name.isEmpty()) { 3 | db_name = "app"; 4 | } 5 | db_uri = "datomic:sql://" + db_name + "?jdbc:sqlite:/usr/storage/sqlite.db"; 6 | for (int i = 0; i < 20; i++) { 7 | Thread.sleep(3000); 8 | System.out.println("Testing connection to database '" + db_name + "'..."); 9 | try { 10 | if (Peer.createDatabase(db_uri)) { 11 | System.out.println("Created database '" + db_name + "'"); 12 | } 13 | try { 14 | new File("/usr/datomic-pro/db_ready").createNewFile(); 15 | } catch (Exception e) { 16 | System.out.println("Could not create file /usr/datomic-pro/db_ready: " + e.getMessage()); 17 | } 18 | System.out.println("Connect using DB URI datomic:sql://" + db_name + "?jdbc:sqlite:"); 19 | System.out.println(" e.g. datomic:sql://app?jdbc:sqlite:./storage/sqlite.db if you mounted ./storage"); 20 | System.exit(0); 21 | } catch (Exception e) { 22 | // System.out.println(e.getMessage()); 23 | } 24 | } 25 | System.out.println("WARNING: Could not create or connect to database " + db_name + " after 10s."); 26 | -------------------------------------------------------------------------------- /image/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/docker-library/repo-info/tree/master/repos/clojure/remote 2 | FROM clojure:temurin-21-tools-deps-1.12.0.1530-bookworm-slim 3 | 4 | RUN apt-get update && apt-get install -y curl unzip && rm -rf /var/lib/apt/lists/* 5 | 6 | # https://docs.datomic.com/setup/pro-setup.html#get-datomic 7 | ARG DATOMIC_VERSION="1.0.7394" 8 | RUN curl https://datomic-pro-downloads.s3.amazonaws.com/${DATOMIC_VERSION}/datomic-pro-${DATOMIC_VERSION}.zip \ 9 | -o datomic-pro.zip \ 10 | && echo "8e3a6334dfc728c1c431dccc537dc88a9d2baf70f29bad5438df9d7c8c7146ae datomic-pro.zip" | sha256sum -c - \ 11 | && unzip datomic-pro.zip \ 12 | && mv datomic-pro-${DATOMIC_VERSION} /usr/datomic-pro \ 13 | || exit 1 14 | 15 | RUN curl -L https://github.com/xerial/sqlite-jdbc/releases/download/3.50.3.0/sqlite-jdbc-3.50.3.0.jar \ 16 | -o sqlite-jdbc-3.50.3.0.jar \ 17 | && echo "a3f53a2aa15ae9425a9e793bbe9c8e5288febeb4b65ef5c1a4e80d4c2045cf08 sqlite-jdbc-3.50.3.0.jar" | sha256sum -c - \ 18 | && mv sqlite-jdbc-3.50.3.0.jar /usr/datomic-pro/lib \ 19 | || exit 1 20 | 21 | FROM clojure:temurin-21-tools-deps-1.12.0.1530-bookworm-slim 22 | 23 | RUN apt-get update && apt-get install -y sqlite3 && rm -rf /var/lib/apt/lists/* 24 | 25 | COPY --from=0 /usr/datomic-pro /usr/datomic-pro 26 | COPY transactor.properties /usr/datomic-pro/config/ 27 | COPY create-db.bsh /usr/create-db.bsh 28 | HEALTHCHECK CMD /usr/bin/env sh -c '[ -f "/usr/datomic-pro/db_ready" ]' || exit 1 29 | COPY start.sh /usr/start.sh 30 | RUN chmod +x /usr/start.sh 31 | 32 | CMD ["/usr/start.sh"] 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Datomic Pro SQLite 2 | 3 | Get started with [Datomic Pro](https://www.datomic.com) quickly on a single machine setup that will take you pretty far. 4 | 5 | Inspired by how Rails 8 now [uses SQLite for production](https://youtu.be/l56IBad-5aQ). 6 | 7 | Links: [GitHub](https://github.com/filipesilva/datomic-pro-sqlite) [DockerHub](https://hub.docker.com/r/filipesilva/datomic-pro-sqlite) 8 | 9 | 10 | ## Quickstart 11 | 12 | - run `docker run -p 4334:4334 -v ./storage:/usr/storage --name datomic-pro-sqlite-demo --rm filipesilva/datomic-pro-sqlite:latest` 13 | - wait until it says `Connect using DB URI datomic:sql://app?jdbc:sqlite:` 14 | - connect from your clojure app or from a `clj` REPL: 15 | ``` clojure 16 | (add-libs {'com.datomic/peer {:mvn/version "1.0.7394"} 17 | 'org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}}) 18 | (require '[datomic.api :as d]) 19 | 20 | (def db-uri "datomic:sql://app?jdbc:sqlite:./storage/sqlite.db") 21 | (def conn (d/connect db-uri)) 22 | 23 | (d/transact conn [{:db/ident :foo}]) 24 | (d/pull (d/db conn) '[*] :foo) 25 | ;; => #:db{:id 17592186045417, :ident :foo} 26 | ``` 27 | 28 | ## Usage 29 | 30 | Start a new container named `datomic-pro-sqlite-demo` that mounts the local `./storage` folder in the container. 31 | Your SQLite database will be here. 32 | You don't have to make this folder, the container will make it for you. 33 | Delete `./storage` and restart the container if you want to wipe the dabatase. 34 | 35 | ```sh 36 | docker run -p 4334:4334 -v ./storage:/usr/storage --name datomic-pro-sqlite-demo --rm filipesilva/datomic-pro-sqlite:latest 37 | ``` 38 | 39 | You should see this output: 40 | ``` 41 | Creating sqlite database at /usr/storage/sqlite.db 42 | Launching with Java options -server -Xms1g -Xmx1g -XX:+UseG1GC -XX:MaxGCPauseMillis=50 43 | System started 44 | Testing connection to database 'app'... 45 | Testing connection to database 'app'... 46 | Testing connection to database 'app'... 47 | Testing connection to database 'app'... 48 | Created database 'app' 49 | Connect using DB URI datomic:sql://app?jdbc:sqlite: 50 | e.g. datomic:sql://app?jdbc:sqlite:./storage/sqlite.db if you mounted ./storage 51 | ``` 52 | 53 | The container will automatically create the `app` db. 54 | You can change this by passing in another name in the env var `DATOMIC_DB` when creating the container (e.g. `-e DATOMIC_DB=blog` before the `filipesilva/datomic-pro-sqlite:latest` image name). 55 | 56 | Start a new clojure repl with `clj` and use the database. 57 | You will need the SQLite driver dependency in addition to Datomic. 58 | 59 | ```clojure 60 | (add-libs {'com.datomic/peer {:mvn/version "1.0.7394"} 61 | 'org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}}) 62 | (require '[datomic.api :as d]) 63 | 64 | (def db-uri "datomic:sql://app?jdbc:sqlite:./storage/sqlite.db") 65 | (def conn (d/connect db-uri)) 66 | 67 | (d/transact conn [{:db/ident :foo}]) 68 | (d/pull (d/db conn) '[*] :foo) 69 | ;; => #:db{:id 17592186045417, :ident :foo} 70 | ``` 71 | 72 | In tests use the in-memory db instead. 73 | 74 | ```clojure 75 | (def test-db-uri (str "datomic:mem://test-" (random-uuid))) 76 | (d/create-database test-db-uri) 77 | (def test-conn (d/connect test-db-uri)) 78 | 79 | (d/pull (d/db test-conn) '[*] :foo) 80 | ;; => #:db{:id nil} 81 | ``` 82 | 83 | 84 | ### Deployment 85 | 86 | If you're running your clojure app in a container remember to also mount `./storage` there. 87 | The Datomic peer library needs access to the SQLite db too. 88 | 89 | You'll want backups of the SQLite db. You can just backup the `./storage` directory at any time. 90 | 91 | [Litestream](https://litestream.io) is a great automated solution that continuously replicates your db to a s3-compatible bucket, and lets you restore it back easily. 92 | This is what the Rails 8 folks recommend. 93 | 94 | Datomic restore and backup also works as usual since all the datomic files are in `/usr/datomic-pro`, you just need to mount the directories you want to use. 95 | This is especially useful if you want to move from SQLite to a different [storage service](https://docs.datomic.com/operation/storage.html). 96 | 97 | To restore and then backup the [Datomic MusicBrainz sample database backup](https://github.com/Datomic/mbrainz-sample): 98 | ``` 99 | # get mbrainz backup 100 | curl https://s3.amazonaws.com/mbrainz/datomic-mbrainz-1968-1973-backup-2017-07-20.tar -o mbrainz.tar 101 | tar -xvf mbrainz.tar 102 | 103 | # mount ./mbrainz at /usr/restore, and a new ./backup directory /usr/backup 104 | docker run -p 4334:4334 -v ./storage:/usr/storage -v ./mbrainz-1968-1973:/usr/restore -v ./backup:/usr/backup --name datomic-pro-sqlite-demo --rm filipesilva/datomic-pro-sqlite:latest 105 | 106 | # in another console, restore the backup with docker exec 107 | docker exec datomic-pro-sqlite-demo /usr/datomic-pro/bin/datomic restore-db file:///usr/restore/ datomic:sql://mbrainz-1968-1973?jdbc:sqlite:/usr/storage/sqlite.db 108 | 109 | # back it up again 110 | docker exec datomic-pro-sqlite-demo /usr/datomic-pro/bin/datomic backup-db datomic:sql://mbrainz-1968-1973?jdbc:sqlite:/usr/storage/sqlite.db file:///usr/backup/ 111 | 112 | ``` 113 | 114 | 115 | ## Development 116 | 117 | Dev scripts are at `bin` and mostly contain the commands in this README for ease of testing. 118 | 119 | Exceptions are: 120 | - `build` to build the image 121 | - `push` to push it to Dockerhub 122 | --------------------------------------------------------------------------------