├── .clj-kondo └── config.edn ├── .cljstyle ├── .editorconfig ├── .github └── workflows │ └── build.yaml ├── .gitignore ├── bb.edn ├── build.clj ├── changelog.adoc ├── deps.edn ├── docker-compose.yaml ├── license ├── readme.adoc ├── src ├── develop │ └── clojure │ │ ├── user.clj │ │ └── xtdb │ │ └── tarantool │ │ └── example.clj ├── main │ ├── clojure │ │ └── xtdb │ │ │ └── tarantool.clj │ └── tarantool │ │ ├── xtdb.lua │ │ └── xtdb │ │ ├── db.lua │ │ ├── init.lua │ │ ├── migrator.lua │ │ ├── model │ │ └── tx_log.lua │ │ ├── response.lua │ │ ├── utils.lua │ │ └── validator.lua └── test │ └── clojure │ └── xtdb │ └── tarantool_test.clj ├── tests.edn └── version.tmpl /.clj-kondo/config.edn: -------------------------------------------------------------------------------- 1 | {:output {:exclude-files 2 | ["src/main/clojure/data_readers.clj" 3 | "src/main/clojure/data_readers.cljc" 4 | "src/main/clojure/data_readers.cljs"]} 5 | 6 | :skip-comments true 7 | 8 | :linters {:consistent-alias 9 | {:aliases {clojure.pprint pprint 10 | clojure.set set 11 | clojure.string str}} 12 | 13 | :unresolved-namespace 14 | {:exclude [user criterium.core]} 15 | 16 | :unresolved-symbol 17 | {:exclude [(cljs.test/are [thrown? thrown-with-msg?]) 18 | (cljs.test/is [thrown? thrown-with-msg?]) 19 | (clojure.test/are [thrown? thrown-with-msg?]) 20 | (clojure.test/is [thrown? thrown-with-msg?])]} 21 | 22 | :unsorted-required-namespaces 23 | {:level :warning} 24 | 25 | :unused-referred-var 26 | {:exclude {cljs.test [deftest is testing use-fixtures] 27 | clojure.test [deftest is testing use-fixtures]}}} 28 | 29 | :hooks {}} 30 | -------------------------------------------------------------------------------- /.cljstyle: -------------------------------------------------------------------------------- 1 | {:files {:ignore #{".clj-kondo" ".idea" ".cpcache" "target" ".git"}} 2 | 3 | :rules {:blank-lines {:max-consecutive 3} 4 | :types {:enabled? false} 5 | :namespaces {:import-break-width 130} 6 | :indentation {:indents {}}}} 7 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | max_line_length = 130 10 | trim_trailing_whitespace = true 11 | 12 | [{*.md, *.adoc}] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - develop 8 | pull_request: 9 | branches: 10 | - master 11 | - develop 12 | 13 | 14 | jobs: 15 | 16 | lint: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v2 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Setup babashka 25 | uses: just-sultanov/setup-babashka@v2 26 | with: 27 | version: '0.6.8' 28 | 29 | - name: Setup clj-kondo 30 | uses: DeLaGuardo/setup-clj-kondo@master 31 | with: 32 | version: '2021.12.01' 33 | 34 | - name: Setup cljstyle 35 | uses: just-sultanov/setup-cljstyle@v1 36 | with: 37 | version: '0.15.0' 38 | 39 | - name: Cache deps 40 | uses: actions/cache@v2 41 | with: 42 | path: | 43 | ~/.m2/repository 44 | ~/.gitlibs 45 | ~/.clojure 46 | ~/.cpcache 47 | key: ${{ runner.os }}-${{ hashFiles('**/deps.edn') }} 48 | 49 | - name: Run linters 50 | run: bb lint 51 | 52 | 53 | 54 | test: 55 | needs: lint 56 | runs-on: ubuntu-latest 57 | steps: 58 | - name: Checkout 59 | uses: actions/checkout@v2 60 | with: 61 | fetch-depth: 0 62 | 63 | - name: Setup environment variables 64 | run: >- 65 | echo 'CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}' >> $GITHUB_ENV; 66 | 67 | - name: Setup openjdk 68 | uses: actions/setup-java@v2 69 | with: 70 | distribution: 'temurin' 71 | java-version: '17' 72 | 73 | - name: Setup clojure 74 | uses: DeLaGuardo/setup-clojure@3.6 75 | with: 76 | cli: latest 77 | 78 | - name: Setup babashka 79 | uses: just-sultanov/setup-babashka@v2 80 | with: 81 | version: '0.6.8' 82 | 83 | - name: Cache deps 84 | uses: actions/cache@v2 85 | with: 86 | path: | 87 | ~/.m2/repository 88 | ~/.gitlibs 89 | ~/.clojure 90 | ~/.cpcache 91 | key: ${{ runner.os }}-${{ hashFiles('**/deps.edn') }} 92 | 93 | - name: Start services 94 | run: bb up 95 | 96 | - name: Run tests 97 | run: bb test 98 | 99 | - name: Stop services 100 | run: bb down 101 | 102 | - name: Upload coverage 103 | run: bash <(curl -s https://codecov.io/bash) -t $CODECOV_TOKEN -f ./coverage/codecov.json 104 | 105 | 106 | 107 | deploy: 108 | runs-on: ubuntu-latest 109 | needs: test 110 | steps: 111 | - name: Checkout 112 | uses: actions/checkout@v2 113 | with: 114 | fetch-depth: 0 115 | 116 | - name: Setup environment variables 117 | run: >- 118 | echo 'CLOJARS_USERNAME=${{ secrets.CLOJARS_USERNAME }}' >> $GITHUB_ENV; 119 | echo 'CLOJARS_PASSWORD=${{ secrets.CLOJARS_PASSWORD }}' >> $GITHUB_ENV; 120 | 121 | - name: Setup openjdk 122 | uses: actions/setup-java@v2 123 | with: 124 | distribution: 'temurin' 125 | java-version: '17' 126 | 127 | - name: Setup clojure 128 | uses: DeLaGuardo/setup-clojure@3.6 129 | with: 130 | cli: latest 131 | 132 | - name: Setup babashka 133 | uses: just-sultanov/setup-babashka@v2 134 | with: 135 | version: '0.6.8' 136 | 137 | - name: Cache deps 138 | uses: actions/cache@v2 139 | with: 140 | path: | 141 | ~/.m2/repository 142 | ~/.gitlibs 143 | ~/.clojure 144 | ~/.cpcache 145 | key: ${{ runner.os }}-${{ hashFiles('**/deps.edn') }} 146 | 147 | - name: Cleanup 148 | run: bb clean 149 | 150 | - name: Run build jar 151 | run: bb jar 152 | 153 | - name: Run deploy 154 | run: bb deploy 155 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.jar 3 | *~ 4 | .classpath 5 | .clj-kondo/.cache 6 | .cljs_node_repl 7 | .cpcache 8 | .idea 9 | .java-version 10 | .lsp/.cache 11 | .lsp/sqlite.db 12 | .nrepl-history 13 | .nrepl-port 14 | .vscode 15 | checkouts 16 | classes 17 | coverage 18 | node_modules 19 | target 20 | build.edn 21 | -------------------------------------------------------------------------------- /bb.edn: -------------------------------------------------------------------------------- 1 | {:min-bb-version 2 | "0.6.8" 3 | 4 | :tasks 5 | {:requires ([babashka.fs :as fs] 6 | [babashka.process :as process] 7 | [clojure.string :as str] 8 | [clojure.pprint :as pprint]) 9 | 10 | :init (do 11 | (defn get-env [s] 12 | (System/getenv s)) 13 | 14 | (defn get-property [s] 15 | (System/getProperty s)) 16 | 17 | (defn pretty-print [x] 18 | (binding [pprint/*print-right-margin* 130] 19 | (pprint/pprint x))) 20 | 21 | (defn execute [command] 22 | (-> command (process/tokenize) (process/process) :out slurp str/trim-newline)) 23 | 24 | (def -zone-id (java.time.ZoneId/of "UTC")) 25 | (def -datetime-formatter java.time.format.DateTimeFormatter/ISO_OFFSET_DATE_TIME) 26 | (def -current-timestamp (java.time.ZonedDateTime/now -zone-id)) 27 | (def -build-timestamp (str (.format -current-timestamp -datetime-formatter))) 28 | (def -build-number (execute "git rev-list HEAD --count")) 29 | (def -git-url (execute "git config --get remote.origin.url")) 30 | (def -git-branch (execute "git rev-parse --abbrev-ref HEAD")) 31 | (def -git-sha (execute "git rev-parse --short HEAD")) 32 | 33 | (def -release? (= "master" -git-branch)) 34 | (def -snapshot? (not -release?)) 35 | (def -deployable? (contains? #{"master" "develop"} -git-branch)) 36 | 37 | (def -version-template (execute "cat version.tmpl")) 38 | (def -version (cond-> (str/replace -version-template "{{build-number}}" -build-number) 39 | -snapshot? (str "-SNAPSHOT"))) 40 | 41 | (def -config 42 | {:version -version 43 | :build-number -build-number 44 | :build-timestamp -build-timestamp 45 | :git-url -git-url 46 | :git-branch -git-branch 47 | :git-sha -git-sha}) 48 | 49 | (def extra-env 50 | {}) 51 | 52 | (defn as-params [params] 53 | (->> params 54 | (seq) 55 | (flatten) 56 | (map (fn [x] 57 | (str/replace (pr-str x) (java.util.regex.Pattern/compile "(\".+\")") "'$1'"))) 58 | (str/join \space))) 59 | 60 | (defn with-params [command] 61 | (->> -config 62 | (as-params) 63 | (str command " ")))) 64 | 65 | :enter (let [{:keys [doc print-doc?] 66 | :or {print-doc? true}} (current-task)] 67 | (when (and print-doc? doc) 68 | (println (str "▸ " doc)))) 69 | 70 | ;;;; 71 | ;; Tasks 72 | ;;;; 73 | 74 | version {:doc "[xtdb-tarantool] Show version" 75 | :print-doc? false 76 | :task (print -version)} 77 | 78 | 79 | config {:doc "[xtdb-tarantool] Show config" 80 | :print-doc? false 81 | :task (pretty-print -config)} 82 | 83 | outdated {:doc "[xtdb-tarantool] Check for outdated dependencies" 84 | :task (clojure (with-params "-T:build outdated"))} 85 | 86 | outdated:upgrade {:doc "[xtdb-tarantool] Upgrade outdated dependencies" 87 | :task (clojure (with-params "-T:build outdated:upgrade"))} 88 | 89 | clean {:doc "[xtdb-tarantool] Cleanup" 90 | :task (clojure (with-params "-T:build clean"))} 91 | 92 | lint {:doc "[xtdb-tarantool] Run linters" 93 | :task (do 94 | (shell "cljstyle check src") 95 | (shell "clj-kondo --lint src"))} 96 | 97 | lint:fix {:doc "[xtdb-tarantool] Run linters & fix" 98 | :task (shell "cljstyle fix src")} 99 | 100 | repl {:doc "[xtdb-tarantool] Run REPL" 101 | :depends [clean] 102 | :task (shell {:extra-env extra-env} (with-params "clojure -T:build repl"))} 103 | 104 | test {:doc "[xtdb-tarantool] Run tests" 105 | :depends [clean] 106 | :task (shell {:extra-env extra-env} (with-params "clojure -T:build test"))} 107 | 108 | test:unit {:doc "[xtdb-tarantool] Run unit tests" 109 | :depends [clean] 110 | :task (shell {:extra-env extra-env} (with-params "clojure -T:build test:unit"))} 111 | 112 | test:integration {:doc "[xtdb-tarantool] Run integration tests" 113 | :depends [clean] 114 | :task (shell {:extra-env extra-env} (with-params "clojure -T:build test:integration"))} 115 | 116 | jar {:doc "[xtdb-tarantool] Run build jar" 117 | :depends [clean] 118 | :task (clojure (with-params "-T:build jar"))} 119 | 120 | install {:doc "[xtdb-tarantool] Install the jar locally" 121 | :task (clojure (with-params "-T:build install"))} 122 | 123 | deploy {:doc "[xtdb-tarantool] Deploy the jar to clojars" 124 | :task (if -deployable? 125 | (clojure (with-params "-T:build deploy")) 126 | (println "Skip deploy..."))} 127 | 128 | up {:doc "[xtdb-tarantool] Start development environment" 129 | :task (shell {:extra-env extra-env} "docker-compose up -d")} 130 | 131 | down {:doc "[xtdb-tarantool] Shutdown development environment" 132 | :task (shell {:extra-env extra-env} "docker-compose down")}}} 133 | -------------------------------------------------------------------------------- /build.clj: -------------------------------------------------------------------------------- 1 | (ns build 2 | (:refer-clojure :exclude [test]) 3 | (:require 4 | [clojure.pprint :as pprint] 5 | [clojure.set :as set] 6 | [clojure.tools.build.util.file :as file] 7 | [org.corfield.build :as bb])) 8 | 9 | 10 | (def defaults 11 | {:src-dirs ["src/main/clojure"] 12 | :resource-dirs ["src/main/resources"] 13 | :lib 'team.sultanov/xtdb-tarantool 14 | :target "target" 15 | :coverage-dir "coverage" 16 | :jar-file "target/xtdb-tarantool.jar" 17 | :build-meta-dir "src/main/resources/xtdb-tarantool"}) 18 | 19 | 20 | 21 | (defn pretty-print 22 | [x] 23 | (binding [pprint/*print-right-margin* 130] 24 | (pprint/pprint x))) 25 | 26 | 27 | 28 | (defn with-defaults 29 | [opts] 30 | (merge defaults opts)) 31 | 32 | 33 | 34 | (defn extract-meta 35 | [opts] 36 | (-> opts 37 | (select-keys [:lib 38 | :version 39 | :build-number 40 | :build-timestamp 41 | :git-url 42 | :git-branch 43 | :git-sha]) 44 | (set/rename-keys {:lib :module}) 45 | (update :module str))) 46 | 47 | 48 | 49 | (defn write-meta 50 | [opts] 51 | (let [dir (:build-meta-dir opts)] 52 | (file/ensure-dir dir) 53 | (->> opts 54 | (extract-meta) 55 | (pretty-print) 56 | (with-out-str) 57 | (spit (format "%s/build.edn" dir))))) 58 | 59 | 60 | 61 | (defn outdated 62 | [opts] 63 | (-> opts 64 | (with-defaults) 65 | (bb/run-task [:nop :outdated]))) 66 | 67 | 68 | 69 | (defn outdated:upgrade 70 | [opts] 71 | (-> opts 72 | (with-defaults) 73 | (bb/run-task [:nop :outdated :outdated/upgrade]))) 74 | 75 | 76 | 77 | (defn clean 78 | [opts] 79 | (-> opts 80 | (with-defaults) 81 | (bb/clean))) 82 | 83 | 84 | 85 | (defn repl 86 | [opts] 87 | (let [opts (with-defaults opts)] 88 | (write-meta opts) 89 | (bb/run-task opts [:test :develop]))) 90 | 91 | 92 | 93 | (defn test 94 | [opts] 95 | (let [opts (with-defaults opts)] 96 | (write-meta opts) 97 | (bb/run-task opts [:test]))) 98 | 99 | 100 | (defn test:unit 101 | [opts] 102 | (let [opts (with-defaults opts)] 103 | (write-meta opts) 104 | (bb/run-task opts [:test :test/unit]))) 105 | 106 | 107 | (defn test:integration 108 | [opts] 109 | (let [opts (with-defaults opts)] 110 | (write-meta opts) 111 | (bb/run-task opts [:test :test/integration]))) 112 | 113 | 114 | 115 | (defn jar 116 | [opts] 117 | (let [opts (with-defaults opts)] 118 | (write-meta opts) 119 | (-> opts 120 | (assoc :scm {:url (:git-url opts) 121 | :tag (:version opts)}) 122 | (bb/jar)))) 123 | 124 | 125 | 126 | (defn install 127 | [opts] 128 | (-> opts 129 | (with-defaults) 130 | (bb/install))) 131 | 132 | 133 | 134 | (defn deploy 135 | [opts] 136 | (-> opts 137 | (with-defaults) 138 | (bb/deploy))) 139 | -------------------------------------------------------------------------------- /changelog.adoc: -------------------------------------------------------------------------------- 1 | == Changelog 2 | 3 | === Released 4 | 5 | ''' 6 | 7 | ==== 0.1.61 (2021-12-04) 8 | 9 | *The first official release* 10 | 11 | ===== Added 12 | 13 | - Tarantool configuration and client API 14 | - Support XTDB tx-log API 15 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src/main/clojure" "src/main/resources"] 2 | 3 | :deps {org.clojure/clojure {:mvn/version "1.10.3"} 4 | org.clojure/tools.logging {:mvn/version "1.1.0"} 5 | com.xtdb/xtdb-core {:mvn/version "1.20.0"} 6 | io.tarantool/cartridge-driver {:mvn/version "0.6.0"}} 7 | 8 | :aliases {:develop {:extra-paths ["src/develop/clojure" "src/develop/resources"] 9 | :extra-deps {nrepl/nrepl {:mvn/version "0.8.3"} 10 | hashp/hashp {:mvn/version "0.2.1"} 11 | integrant/integrant {:mvn/version "0.8.0"} 12 | integrant/repl {:mvn/version "0.3.2"} 13 | org.slf4j/slf4j-simple {:mvn/version "1.7.32"} 14 | com.xtdb/xtdb-http-server {:mvn/version "1.20.0"}} 15 | :main-opts ["--main" "nrepl.cmdline"]} 16 | 17 | :test {:extra-paths ["src/test/clojure" "src/test/resources"] 18 | :extra-deps {lambdaisland/kaocha {:mvn/version "1.60.945"} 19 | lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}} 20 | :main-opts ["--main" "kaocha.runner"]} 21 | 22 | :test/unit {:main-opts ["--main" "kaocha.runner" "--focus" "unit"]} 23 | 24 | :test/integration {:main-opts ["--main" "kaocha.runner" "--focus" "integration"]} 25 | 26 | :build {:extra-paths ["."] 27 | :extra-deps {io.github.seancorfield/build-clj {:git/tag "v0.6.0" :git/sha "2451bea"}} 28 | :ns-default build} 29 | 30 | :nop {:extra-deps {org.slf4j/slf4j-nop {:mvn/version "1.7.32"}}} 31 | 32 | :outdated {:extra-deps {com.github.liquidz/antq {:mvn/version "1.3.0"}} 33 | :main-opts ["--main" "antq.core"]} 34 | 35 | :outdated/upgrade {:main-opts ["--main" "antq.core" "--upgrade" "--force"]}}} 36 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | tarantool-node1: 6 | image: tarantool/tarantool:2.8.2 7 | container_name: tarantool-node1 8 | hostname: tarantool-node1 9 | environment: 10 | - TARANTOOL_USER_NAME=root 11 | - TARANTOOL_USER_PASSWORD=root 12 | command: tarantool /opt/tarantool/xtdb.lua 13 | networks: 14 | - net 15 | ports: 16 | - "3301:3301" 17 | volumes: 18 | # operational data (snapshots, xlogs and vinyl runs) 19 | - "/tmp/tarantool/data:/var/lib/tarantool" 20 | # application code 21 | - "./src/main/tarantool:/opt/tarantool" 22 | 23 | 24 | networks: 25 | net: 26 | driver: bridge 27 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 sultanov.team 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /readme.adoc: -------------------------------------------------------------------------------- 1 | image:https://img.shields.io/github/license/sultanov-team/xtdb-tarantool[license,link=license] 2 | image:https://codecov.io/gh/sultanov-team/xtdb-tarantool/branch/master/graph/badge.svg?token=3VpsOfOpH5)[codecov,link=https://codecov.io/gh/sultanov-team/xtdb-tarantool] 3 | image:https://github.com/sultanov-team/xtdb-tarantool/workflows/build/badge.svg[build] 4 | 5 | image:https://img.shields.io/clojars/v/team.sultanov/xtdb-tarantool.svg[clojars,link=https://clojars.org/team.sultanov/xtdb-tarantool] 6 | 7 | == XTDB Tarantool 8 | 9 | https://www.xtdb.com[XTDB] module allows you to use https://www.tarantool.io/[Tarantool] (in-memory computing platform). 10 | 11 | **Status: ** Alpha. 12 | The design and prototyping stage. 13 | 14 | === Installation 15 | 16 | Add the following dependency in your project: 17 | 18 | [source,clojure] 19 | ---- 20 | ;; project.clj or build.boot 21 | [team.sultanov/xtdb-tarantool "0.1.61"] 22 | 23 | ;; deps.edn 24 | team.sultanov/xtdb-tarantool {:mvn/version "0.1.61"} 25 | ---- 26 | 27 | === Usage 28 | 29 | *Requirements* 30 | 31 | First you need to configure the Tarantool: 32 | 33 | - link:docker-compose.yaml[Docker compose] 34 | - link:src/main/tarantool/xtdb.lua[Tarantool configuration] 35 | 36 | [source,clojure] 37 | ---- 38 | ;; src/develop/clojure/xtdb/tarantool/example.clj 39 | 40 | (ns xtdb.tarantool.example 41 | (:require 42 | [clojure.tools.namespace.repl :as tools.repl] 43 | [integrant.core :as ig] 44 | [integrant.repl :as ig.repl] 45 | [integrant.repl.state :as ig.repl.state] 46 | [xtdb.api :as xt] 47 | [xtdb.tarantool :as tnt]) 48 | (:import 49 | (java.io 50 | Closeable) 51 | (java.time 52 | Duration))) 53 | 54 | 55 | (tools.repl/set-refresh-dirs "src/dev/clojure") 56 | 57 | 58 | (defn config 59 | [] 60 | {::xtdb-tnt {::tnt-client {:xtdb/module 'xtdb.tarantool/->tnt-client 61 | :username "root" 62 | :password "root"} 63 | :xtdb/tx-log {:xtdb/module 'xtdb.tarantool/->tx-log 64 | :tnt-client ::tnt-client 65 | :poll-wait-duration (Duration/ofSeconds 5)} 66 | :xtdb.http-server/server {:read-only? true 67 | :server-label "[xtdb-tarantool] Console Demo"}}}) 68 | 69 | 70 | (defn prep 71 | [] 72 | (ig.repl/set-prep! config)) 73 | 74 | 75 | (defn go 76 | [] 77 | (prep) 78 | (ig.repl/go)) 79 | 80 | 81 | (def halt ig.repl/halt) 82 | (def reset-all ig.repl/reset-all) 83 | 84 | 85 | (defn system 86 | [] 87 | ig.repl.state/system) 88 | 89 | 90 | (defmethod ig/init-key ::xtdb-tnt [_ config] 91 | (xt/start-node config)) 92 | 93 | 94 | (defmethod ig/halt-key! ::xtdb-tnt [_ ^Closeable node] 95 | (tnt/close node)) 96 | 97 | 98 | (comment 99 | 100 | (reset-all) 101 | (halt) 102 | (go) 103 | ;; open http://localhost:3000/ 104 | 105 | 106 | (def node 107 | (::xtdb-tnt (system))) 108 | 109 | 110 | (xt/submit-tx node [[::xt/put {:xt/id "xtdb-tarantool", :user/email "ilshat@sultanov.team"}]]) 111 | ;; => #:xtdb.api{:tx-id 1, :tx-time #inst"2021-12-04T01:27:15.641-00:00"} 112 | 113 | 114 | (xt/q (xt/db node) '{:find [e] 115 | :where [[e :user/email "ilshat@sultanov.team"]]}) 116 | ;; => #{["xtdb-tarantool"]} 117 | 118 | 119 | (xt/q (xt/db node) 120 | '{:find [(pull ?e [*])] 121 | :where [[?e :xt/id "xtdb-tarantool"]]}) 122 | ;; => #{[{:user/email "ilshat@sultanov.team", :xt/id "xtdb-tarantool"}]} 123 | 124 | 125 | (def history (xt/entity-history (xt/db node) "xtdb-tarantool" :desc {:with-docs? true})) 126 | ;; => [#:xtdb.api{:tx-time #inst"2021-12-04T01:31:14.080-00:00", 127 | ;; :tx-id 2, 128 | ;; :valid-time #inst"2021-12-04T01:31:14.080-00:00", 129 | ;; :content-hash #xtdb/id"d0eb040d39fbdaa8699d867bc9fb9aa244b8e154", 130 | ;; :doc {:user/email "ilshat@sultanov.team", :xt/id "xtdb-tarantool"}}] 131 | 132 | 133 | (->> (map ::xt/doc history) 134 | (filterv (comp (partial = "ilshat@sultanov.team") :user/email))) 135 | ;; => [{:user/email "ilshat@sultanov.team", :xt/id "xtdb-tarantool"}] 136 | ) 137 | ---- 138 | 139 | === Roadmap 140 | 141 | - [x] Add tx-log 142 | - [ ] Add kv-store 143 | - [ ] Add document-store 144 | - [ ] Add logging 145 | - [ ] Improve tests 146 | - [ ] Add automatic tarantool configuration (in-memory / vinyl) 147 | - [ ] Add an example of using a tarantool cartridge 148 | - [ ] Add an example of using a tarantool kubernetes operator (single node / cluster / with sharding) 149 | - [ ] Add rockspec and publish tarantool configuration to https://luarocks.org[luarocks]? 150 | 151 | === Workflow 152 | 153 | ==== Development 154 | 155 | [source,bash] 156 | ---- 157 | # run tarantool 158 | $ bb up 159 | 160 | # stop tarantool 161 | $ bb down 162 | 163 | # run repl 164 | $ bb repl 165 | 166 | # check for outdated dependencies 167 | $ bb outdated 168 | 169 | # upgrade outdated dependencies 170 | $ bb outdated:upgrade 171 | ---- 172 | 173 | ==== Testing 174 | 175 | [source,bash] 176 | ---- 177 | # run linters 178 | $ bb lint 179 | 180 | # run linters and fix issues 181 | $ bb lint:fix 182 | 183 | # run only unit tests 184 | $ bb test:unit 185 | 186 | # run only integration tests (requires a running tarantool) 187 | $ bb test:integration 188 | 189 | # run all tests 190 | $ bb test 191 | ---- 192 | 193 | ==== Build & deploy 194 | 195 | [source,bash] 196 | ---- 197 | # build jar 198 | $ bb jar 199 | 200 | # install locally 201 | $ bb install 202 | 203 | # deploy to clojars 204 | $ bb deploy 205 | ---- 206 | 207 | ==== Other tasks 208 | 209 | [source,bash] 210 | ---- 211 | $ bb tasks 212 | ---- 213 | 214 | === License 215 | 216 | Copyright © 2021 sultanov.team 217 | -------------------------------------------------------------------------------- /src/develop/clojure/user.clj: -------------------------------------------------------------------------------- 1 | (ns user 2 | "Development helper functions." 3 | (:require 4 | [hashp.core])) 5 | 6 | 7 | (defmacro jit 8 | "Just in time loading of dependencies." 9 | [sym] 10 | `(requiring-resolve '~sym)) 11 | -------------------------------------------------------------------------------- /src/develop/clojure/xtdb/tarantool/example.clj: -------------------------------------------------------------------------------- 1 | (ns xtdb.tarantool.example 2 | (:require 3 | [clojure.tools.namespace.repl :as tools.repl] 4 | [integrant.core :as ig] 5 | [integrant.repl :as ig.repl] 6 | [integrant.repl.state :as ig.repl.state] 7 | [xtdb.api :as xt] 8 | [xtdb.tarantool :as tnt]) 9 | (:import 10 | (java.io 11 | Closeable) 12 | (java.time 13 | Duration))) 14 | 15 | 16 | (tools.repl/set-refresh-dirs "src/dev/clojure") 17 | 18 | 19 | (defn config 20 | [] 21 | {::xtdb-tnt {::tnt-client {:xtdb/module 'xtdb.tarantool/->tnt-client 22 | :username "root" 23 | :password "root"} 24 | :xtdb/tx-log {:xtdb/module 'xtdb.tarantool/->tx-log 25 | :tnt-client ::tnt-client 26 | :poll-wait-duration (Duration/ofSeconds 5)} 27 | :xtdb.http-server/server {:read-only? true 28 | :server-label "[xtdb-tarantool] Console Demo"}}}) 29 | 30 | 31 | (defn prep 32 | [] 33 | (ig.repl/set-prep! config)) 34 | 35 | 36 | (defn go 37 | [] 38 | (prep) 39 | (ig.repl/go)) 40 | 41 | 42 | (def halt ig.repl/halt) 43 | (def reset-all ig.repl/reset-all) 44 | 45 | 46 | (defn system 47 | [] 48 | ig.repl.state/system) 49 | 50 | 51 | (defmethod ig/init-key ::xtdb-tnt [_ config] 52 | (xt/start-node config)) 53 | 54 | 55 | (defmethod ig/halt-key! ::xtdb-tnt [_ ^Closeable node] 56 | (tnt/close node)) 57 | 58 | 59 | (comment 60 | 61 | (reset-all) 62 | (halt) 63 | (go) 64 | ;; open http://localhost:3000/ 65 | 66 | 67 | (def node 68 | (::xtdb-tnt (system))) 69 | 70 | (xt/submit-tx node [[::xt/put {:xt/id "xtdb-tarantool", :user/email "ilshat@sultanov.team"}]]) 71 | ;; => #:xtdb.api{:tx-id 1, :tx-time #inst"2021-12-04T01:27:15.641-00:00"} 72 | 73 | 74 | (xt/q (xt/db node) '{:find [e] 75 | :where [[e :user/email "ilshat@sultanov.team"]]}) 76 | ;; => #{["xtdb-tarantool"]} 77 | 78 | 79 | (xt/q (xt/db node) 80 | '{:find [(pull ?e [*])] 81 | :where [[?e :xt/id "xtdb-tarantool"]]}) 82 | ;; => #{[{:user/email "ilshat@sultanov.team", :xt/id "xtdb-tarantool"}]} 83 | 84 | 85 | 86 | (def history (xt/entity-history (xt/db node) "xtdb-tarantool" :desc {:with-docs? true})) 87 | ;; => [#:xtdb.api{:tx-time #inst"2021-12-04T01:31:14.080-00:00", 88 | ;; :tx-id 2, 89 | ;; :valid-time #inst"2021-12-04T01:31:14.080-00:00", 90 | ;; :content-hash #xtdb/id"d0eb040d39fbdaa8699d867bc9fb9aa244b8e154", 91 | ;; :doc {:user/email "ilshat@sultanov.team", :xt/id "xtdb-tarantool"}}] 92 | 93 | 94 | (->> (map ::xt/doc history) 95 | (filterv (comp (partial = "ilshat@sultanov.team") :user/email))) 96 | ;; => [{:user/email "ilshat@sultanov.team", :xt/id "xtdb-tarantool"}] 97 | ) 98 | -------------------------------------------------------------------------------- /src/main/clojure/xtdb/tarantool.clj: -------------------------------------------------------------------------------- 1 | (ns xtdb.tarantool 2 | (:require 3 | [clojure.spec.alpha :as s] 4 | [clojure.string :as str] 5 | [juxt.clojars-mirrors.nippy.v3v1v1.taoensso.nippy :as nippy] 6 | [xtdb.api :as xt] 7 | [xtdb.db :as db] 8 | [xtdb.io :as xio] 9 | [xtdb.system :as system] 10 | [xtdb.tx.event :as xte] 11 | [xtdb.tx.subscribe :as tx-sub]) 12 | (:import 13 | (clojure.lang 14 | PersistentHashSet 15 | PersistentList 16 | PersistentTreeSet 17 | PersistentVector) 18 | (io.tarantool.driver.api 19 | TarantoolClient 20 | TarantoolClientFactory) 21 | (io.tarantool.driver.api.retry 22 | TarantoolRequestRetryPolicies$AttemptsBoundRetryPolicyFactory$Builder) 23 | (io.tarantool.driver.mappers 24 | DefaultMessagePackMapper 25 | DefaultMessagePackMapperFactory) 26 | (java.io 27 | Closeable) 28 | (java.lang 29 | AutoCloseable) 30 | (java.time 31 | Duration 32 | Instant) 33 | (java.time.temporal 34 | ChronoUnit) 35 | (java.util 36 | ArrayList 37 | Date 38 | List) 39 | (java.util.concurrent 40 | CompletableFuture) 41 | (java.util.function 42 | Function 43 | UnaryOperator))) 44 | 45 | 46 | (set! *warn-on-reflection* true) 47 | 48 | 49 | ;; 50 | ;; Helper functions 51 | ;; 52 | 53 | (defn to-inst 54 | "Converts microseconds to `java.util.Date`." 55 | [^long ts] 56 | (-> (Instant/EPOCH) 57 | (.plus ts ChronoUnit/MICROS) 58 | (Date/from))) 59 | 60 | 61 | (defprotocol ITarantoolCallFnArgument 62 | "This protocol is used to simplify manipulations with the arguments of the function defined in the Tarantool instance." 63 | :extend-via-metadata true 64 | (-prepare-fn-args [argument])) 65 | 66 | 67 | (extend-protocol ITarantoolCallFnArgument 68 | PersistentList 69 | (-prepare-fn-args [coll] (ArrayList. coll)) 70 | 71 | PersistentVector 72 | (-prepare-fn-args [coll] (ArrayList. coll)) 73 | 74 | PersistentHashSet 75 | (-prepare-fn-args [coll] (ArrayList. coll)) 76 | 77 | PersistentTreeSet 78 | (-prepare-fn-args [coll] (ArrayList. coll)) 79 | 80 | Object 81 | (-prepare-fn-args [object] (-prepare-fn-args [object])) 82 | 83 | nil 84 | (-prepare-fn-args [_] (ArrayList.))) 85 | 86 | 87 | (defn prepare-fn-args 88 | "Prepares the list of arguments for executing the function defined in the Tarantool instance." 89 | (^List [] (-prepare-fn-args nil)) 90 | (^List [x] (-prepare-fn-args x))) 91 | 92 | 93 | (defn serialize 94 | [x] 95 | (nippy/freeze-to-string x)) 96 | 97 | 98 | (defn deserialize 99 | [x] 100 | (nippy/thaw-from-string x)) 101 | 102 | 103 | 104 | ;; 105 | ;; Default mappers 106 | ;; 107 | 108 | (def ^{:doc "Modification-safe instance of the messagepack mapper. 109 | The instance contains converters for simple types and complex types `java.util.Map` and `java.util.List`."} 110 | default-complex-types-mapper 111 | (.defaultComplexTypesMapper (DefaultMessagePackMapperFactory/getInstance))) 112 | 113 | 114 | (def ^{:doc "Modification-safe instance of the messagepack mapper. 115 | The instance already contains converters for simple types."} 116 | default-simple-type-mapper 117 | (.defaultSimpleTypeMapper (DefaultMessagePackMapperFactory/getInstance))) 118 | 119 | 120 | 121 | ;; 122 | ;; Default builders 123 | ;; 124 | 125 | (defn default-exception-handler 126 | "Returns a default exception handler." 127 | [] 128 | (reify 129 | Function 130 | (apply [_ e] 131 | (str/includes? (.getMessage ^Exception e) "Unsuccessful attempt")))) 132 | 133 | 134 | (defn default-request-retry-policy 135 | "Returns a default request retry policy." 136 | [{:keys [^long delay] 137 | :or {delay 300}}] 138 | (reify 139 | UnaryOperator 140 | (apply [_ policy] 141 | (.withDelay ^TarantoolRequestRetryPolicies$AttemptsBoundRetryPolicyFactory$Builder policy delay)))) 142 | 143 | 144 | 145 | ;; 146 | ;; Tarantool API 147 | ;; 148 | 149 | (defn close 150 | "Closes the component." 151 | [^AutoCloseable x] 152 | (.close x)) 153 | 154 | 155 | (defn execute 156 | "Execute a function defined on the Tarantool instance. 157 | Params: 158 | * tnt-client - instance of `io.tarantool.driver.api.TarantoolClient` 159 | * mapper - mapper for arguments object-to-MessagePack entity conversion and result values conversion 160 | * fn-name - function name, must not be null or empty 161 | * fn-args - function arguments (optional)" 162 | ([^TarantoolClient tnt-client ^DefaultMessagePackMapper mapper ^String fn-name] 163 | (execute tnt-client mapper fn-name nil)) 164 | ([^TarantoolClient tnt-client ^DefaultMessagePackMapper mapper ^String fn-name fn-args] 165 | (-> ^CompletableFuture (.call tnt-client fn-name (prepare-fn-args fn-args) mapper) 166 | (.get) 167 | (first)))) 168 | 169 | 170 | (defn get-box-info 171 | [^TarantoolClient tnt-client ^DefaultMessagePackMapper mapper] 172 | (execute tnt-client mapper "box.info")) 173 | 174 | 175 | 176 | ;; 177 | ;; Tarantool client 178 | ;; 179 | 180 | ;; Specs 181 | 182 | (s/def ::host ::system/string) 183 | (s/def ::port ::system/pos-int) 184 | (s/def ::username ::system/string) 185 | (s/def ::password ::system/string) 186 | (s/def ::retries ::system/pos-int) 187 | (s/def ::exception-handler #(instance? Function %)) 188 | (s/def ::request-retry-policy #(instance? UnaryOperator %)) 189 | 190 | 191 | ;; Component 192 | 193 | (defn ^TarantoolClient ->tnt-client 194 | "Returns a tarantool client as a component of the xtdb.system." 195 | {::system/args {:host {:doc "Host" 196 | :spec ::host 197 | :default "localhost" 198 | :required? true} 199 | :port {:doc "Port" 200 | :spec ::port 201 | :default 3301 202 | :required? true} 203 | :username {:doc "Username" 204 | :spec ::username 205 | :required? true} 206 | :password {:doc "Password" 207 | :spec ::password 208 | :required? true} 209 | :retries {:doc "The number of retry attempts for each request" 210 | :spec ::retries 211 | :default 3 212 | :required? true} 213 | :exception-handler {:doc "Function checking whether the given exception may be retried" 214 | :spec ::exception-handler 215 | :default (default-exception-handler) 216 | :required? true} 217 | :request-retry-policy {:doc "Retry policy that performs unbounded number of attempts. If the exception check passes, the policy returns true" 218 | :spec ::request-retry-policy 219 | :default (default-request-retry-policy {:delay 300}) 220 | :required? true}}} 221 | [{:keys [^String host ^long port ^String username ^String password ^long retries ^Function exception-handler ^UnaryOperator request-retry-policy]}] 222 | (let [tnt-client (-> (TarantoolClientFactory/createClient) 223 | (.withAddress host port) 224 | (.withCredentials username password) 225 | (.build))] 226 | (-> (TarantoolClientFactory/configureClient tnt-client) 227 | (.withRetryingByNumberOfAttempts retries exception-handler request-retry-policy) 228 | (.build)))) 229 | 230 | 231 | 232 | ;; 233 | ;; API tx-log 234 | ;; 235 | 236 | (defn submit-tx 237 | "Submits a `tx` to the `tx-log` and returns an instance of `clojure.lang.Delay`." 238 | [{:keys [tnt-client mapper]} tx-events] 239 | (let [{:strs [status data]} (execute tnt-client mapper "xtdb.tx_log.submit_tx" (serialize tx-events))] 240 | (when (= 201 status) 241 | (let [{:strs [tx_id tx_time]} data] 242 | (delay {::xt/tx-id tx_id 243 | ::xt/tx-time (to-inst tx_time)}))))) 244 | 245 | 246 | (defn open-tx-log* 247 | "Returns the `tx-log` as a lazy sequence." 248 | [{:as tx-log :keys [tnt-client mapper]} after-tx-id] 249 | (let [txs (lazy-seq 250 | (let [{:strs [status data]} (execute tnt-client mapper "xtdb.tx_log.open_tx_log" after-tx-id)] 251 | (when (= 200 status) 252 | (let [txs (map (fn [{:strs [tx_id tx_time tx_events]}] 253 | {::xt/tx-id tx_id 254 | ::xt/tx-time (to-inst tx_time) 255 | ::xte/tx-events (deserialize tx_events)}) 256 | data) 257 | last-tx-id (::xt/tx-id (last txs))] 258 | (cons txs (open-tx-log* tx-log last-tx-id))))))] 259 | (mapcat identity txs))) 260 | 261 | 262 | (defn open-tx-log 263 | "Returns the `tx-log` as an instance of `xtdb.api.Cursor`." 264 | [tx-log after-tx-id] 265 | (xio/->cursor (constantly false) (open-tx-log* tx-log after-tx-id))) 266 | 267 | 268 | (defn subscribe 269 | "Subscribe to the `tx-log` (via long-polling) using a `poll-wait-duration`." 270 | [{:as tx-log :keys [poll-wait-duration]} after-tx-id f] 271 | (tx-sub/handle-polling-subscription tx-log after-tx-id {:poll-sleep-duration poll-wait-duration} f)) 272 | 273 | 274 | (defn latest-submitted-tx 275 | "Returns the latest submitted `tx-id` if the `tx-log` is not empty. Otherwise, returns `nil`." 276 | [{:keys [tnt-client mapper]}] 277 | (when-let [{:strs [status data]} (execute tnt-client mapper "xtdb.tx_log.latest_submitted_tx")] 278 | (when (= 200 status) 279 | (let [{:strs [tx_id]} data] 280 | {::xt/tx-id tx_id})))) 281 | 282 | 283 | (defrecord TarantoolTxLog 284 | [^TarantoolClient tnt-client ^DefaultMessagePackMapper mapper ^Duration poll-wait-duration] 285 | db/TxLog 286 | (submit-tx [tx-log tx-events] (submit-tx tx-log tx-events)) 287 | (open-tx-log [tx-log after-tx-id] (open-tx-log tx-log after-tx-id)) 288 | (subscribe [tx-log after-tx-id f] (subscribe tx-log after-tx-id f)) 289 | (latest-submitted-tx [tx-log] (latest-submitted-tx tx-log)) 290 | 291 | Closeable 292 | (close [_] (close tnt-client))) 293 | 294 | 295 | 296 | ;; Specs 297 | 298 | (s/def ::mapper #(instance? DefaultMessagePackMapper %)) 299 | (s/def ::duration ::system/duration) 300 | 301 | 302 | 303 | ;; Component 304 | 305 | (defn ->tx-log 306 | {::system/args {:mapper {:doc "Default implementation of MessagePackObjectMapper and MessagePackValueMapper" 307 | :spec ::mapper 308 | :default default-complex-types-mapper 309 | :required? true} 310 | :poll-wait-duration {:doc "How long to wait when polling the Tarantool instance" 311 | :spec ::duration 312 | :default (Duration/ofSeconds 1) 313 | :required? true}} 314 | ::system/deps {:tnt-client `->tnt-client}} 315 | [opts] 316 | (map->TarantoolTxLog opts)) 317 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb.lua: -------------------------------------------------------------------------------- 1 | box.cfg { 2 | listen = 3301, 3 | log_level = 7, -- debug 4 | memtx_memory = 128 * 1024 * 1024 -- 128 Mb 5 | } 6 | 7 | local config = { 8 | spaces = { 9 | tx_log = 'xtdb_tx_log', 10 | kv_store = 'xtdb_kv_store', 11 | document_store = 'xtdb_document_store' 12 | } 13 | } 14 | 15 | xtdb = require('xtdb.init').setup(config) 16 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb/db.lua: -------------------------------------------------------------------------------- 1 | local db = {} 2 | 3 | function db.configure(config) 4 | local api = {} 5 | 6 | local tx_log = require('xtdb.model.tx_log').model(config) 7 | 8 | function api.create_db() 9 | box.once('schema', function() 10 | 11 | -- 12 | -- tx_log 13 | -- 14 | 15 | box.schema.sequence.create(tx_log.TX_ID_INDEX_SEQ, { 16 | start = 1, 17 | min = 1, 18 | step = 1, 19 | if_not_exists = true 20 | }) 21 | 22 | local tx_log_space = box.schema.space.create(tx_log.SPACE_NAME, { 23 | if_not_exists = true 24 | }) 25 | 26 | tx_log_space:format({ 27 | { name = tx_log.TX_ID_FIELD, type = 'unsigned' }, 28 | { name = tx_log.TX_TIME_FIELD, type = 'unsigned' }, 29 | { name = tx_log.TX_EVENTS_FIELD, type = 'string' } 30 | }) 31 | 32 | tx_log_space:create_index(tx_log.TX_ID_INDEX, { 33 | parts = { tx_log.TX_ID_FIELD }, 34 | sequence = tx_log.TX_ID_INDEX_SEQ, 35 | if_not_exists = true 36 | }) 37 | 38 | end) 39 | end 40 | 41 | function api.truncate_db() 42 | tx_log.get_space():truncate() 43 | end 44 | 45 | return api 46 | 47 | end 48 | 49 | return db 50 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb/init.lua: -------------------------------------------------------------------------------- 1 | local xtdb = {} 2 | 3 | local db = require('xtdb.db') 4 | local log = require('log') 5 | local migrator = require('xtdb.migrator') 6 | local response = require('xtdb.response') 7 | local utils = require('xtdb.utils') 8 | local validator = require('xtdb.validator') 9 | 10 | function xtdb.setup(config) 11 | local api = { 12 | db = {}, 13 | tx_log = {} 14 | } 15 | 16 | -- 17 | -- Bootstrap 18 | -- 19 | 20 | config = validator.validate(config) 21 | db.configure(config).create_db() 22 | migrator.migrate(config) 23 | 24 | 25 | 26 | -- 27 | -- Models 28 | -- 29 | 30 | local tx_log = require('xtdb.model.tx_log').model(config) 31 | 32 | 33 | 34 | -- 35 | -- API methods 36 | -- 37 | 38 | -- 39 | -- [db] truncate 40 | -- 41 | function api.db.truncate() 42 | db.configure(config).truncate_db() 43 | end 44 | 45 | 46 | 47 | -- 48 | -- [tx_log] submit_tx 49 | -- 50 | 51 | function api.tx_log.submit_tx(tx_events) 52 | -- if tx_events is valid 53 | if validator.not_blank_string(tx_events) then 54 | local tx = tx_log.submit_tx(tx_events) 55 | local res = { 56 | tx_id = tx[tx_log.TX_ID], 57 | tx_time = tx[tx_log.TX_TIME] 58 | } 59 | log.info("[tx_log.submit_tx]: `tx`=%s", utils.to_json(res)) 60 | return response.success(response.CREATED, res) 61 | end 62 | 63 | -- if tx_events is not valid 64 | log.error("[tx_log.submit_tx]: BAD REQUEST - `tx_events`=%s", tx_events) 65 | return response.failure(response.BAD_REQUEST) 66 | end 67 | 68 | 69 | 70 | -- 71 | -- [tx_log] open_tx_log 72 | -- 73 | 74 | function api.tx_log.open_tx_log(after_tx_id) 75 | -- if after_tx_id is not valid 76 | if validator.is_some(after_tx_id) and not validator.is_pos(after_tx_id) then 77 | log.error("[tx_log.open_tx_log]: BAD REQUEST - `after_tx_id`=%s", after_tx_id) 78 | return response.failure(response.BAD_REQUEST) 79 | end 80 | 81 | -- if after_tx_id is nil 82 | if validator.is_nil(after_tx_id) then 83 | log.warn("[tx_log.open_tx_log]: `after_tx_id`=0 (instead of %s)", after_tx_id) 84 | after_tx_id = 0 85 | end 86 | 87 | -- get txs 88 | local txs = tx_log.open_tx_log(after_tx_id) 89 | 90 | -- if tx_log is empty 91 | if validator.is_empty(txs) then 92 | log.error("[tx_log.open_tx_log]: `after_tx_id`=%s, `txs`=[]", after_tx_id) 93 | return response.failure(response.NOT_FOUND) 94 | end 95 | 96 | -- if tx_log is not empty 97 | local res = utils.map(function(tx) 98 | return { 99 | tx_id = tx[tx_log.TX_ID], 100 | tx_time = tx[tx_log.TX_TIME], 101 | tx_events = tx[tx_log.TX_EVENTS] 102 | } 103 | end, txs) 104 | log.info("[tx_log.open_tx_log]: `after_tx_id`=%s, `txs`=%s", after_tx_id, utils.to_json(res)) 105 | return response.success(response.OK, res) 106 | end 107 | 108 | 109 | 110 | -- 111 | -- [tx_log] latest_submitted_tx 112 | -- 113 | 114 | function api.tx_log.latest_submitted_tx() 115 | local tx = tx_log.latest_submitted_tx() 116 | 117 | -- if tx is not nil 118 | if validator.is_some(tx) then 119 | local res = { 120 | tx_id = tx[tx_log.TX_ID] 121 | } 122 | log.info("[tx_log.latest_submitted_tx]: `tx`=%s", utils.to_json(res)) 123 | return response.success(response.OK, res) 124 | end 125 | 126 | -- if tx is nil 127 | log.error("[tx_log.latest_submitted_tx]: `tx`=nil") 128 | return response.failure(response.NOT_FOUND) 129 | end 130 | 131 | return api 132 | 133 | end 134 | 135 | return xtdb 136 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb/migrator.lua: -------------------------------------------------------------------------------- 1 | local migrator = {} 2 | 3 | function migrator.migrate(config) 4 | 5 | if box.cfg.read_only == false then 6 | box.once('20211202_xtdb_example_migration', function() 7 | -- put migrations with box.once here 8 | end) 9 | end 10 | 11 | end 12 | 13 | return migrator 14 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb/model/tx_log.lua: -------------------------------------------------------------------------------- 1 | local tx_log = {} 2 | 3 | local utils = require('xtdb.utils') 4 | local validator = require('xtdb.validator') 5 | 6 | function tx_log.model(config) 7 | local model = {} 8 | 9 | -- 10 | -- Definition 11 | -- 12 | 13 | -- space 14 | model.SPACE_NAME = config.spaces.tx_log.name 15 | 16 | -- field order 17 | model.TX_ID = 1 18 | model.TX_TIME = 2 19 | model.TX_EVENTS = 3 20 | 21 | -- field names 22 | model.TX_ID_FIELD = 'tx_id' 23 | model.TX_TIME_FIELD = 'tx_time' 24 | model.TX_EVENTS_FIELD = 'tx_events' 25 | 26 | -- sequences 27 | model.TX_ID_INDEX_SEQ = model.SPACE_NAME .. '_tx_id_index_seq' 28 | 29 | -- indexes 30 | model.TX_ID_INDEX = model.SPACE_NAME .. '_tx_id_index' 31 | 32 | 33 | 34 | -- 35 | -- Helper functions 36 | -- 37 | 38 | function model.get_space() 39 | return box.space[model.SPACE_NAME] 40 | end 41 | 42 | 43 | 44 | 45 | -- 46 | -- API 47 | -- 48 | 49 | function model.submit_tx(tx_events) 50 | return model.get_space():insert({ 51 | [model.TX_ID] = nil, 52 | [model.TX_TIME] = utils.now(), 53 | [model.TX_EVENTS] = tx_events 54 | }) 55 | end 56 | 57 | function model.open_tx_log(after_tx_id) 58 | return model.get_space():select({ [model.TX_ID] = after_tx_id }, 'GT') -- TODO: add limits? 59 | end 60 | 61 | function model.latest_submitted_tx() 62 | return model.get_space().index[model.TX_ID_INDEX]:max() 63 | end 64 | 65 | return model 66 | 67 | end 68 | 69 | return tx_log 70 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb/response.lua: -------------------------------------------------------------------------------- 1 | local response = {} 2 | 3 | -- success responses 4 | 5 | response.OK = 200 6 | response.CREATED = 201 7 | 8 | 9 | 10 | -- error responses 11 | response.BAD_REQUEST = 400 12 | response.NOT_FOUND = 404 13 | 14 | 15 | 16 | -- response codes 17 | response.CODES = { 18 | 19 | -- success responses 20 | [response.OK] = 'Success', 21 | [response.CREATED] = 'Created', 22 | 23 | 24 | -- error responses 25 | [response.BAD_REQUEST] = 'Bad request', 26 | [response.NOT_FOUND] = 'Not found' 27 | 28 | } 29 | 30 | function response.failure(code, error) 31 | return { 32 | status = code, 33 | message = response.CODES[code], 34 | error = error 35 | } 36 | end 37 | 38 | function response.success(code, data) 39 | return { 40 | status = code, 41 | message = response.CODES[code], 42 | data = data 43 | } 44 | end 45 | 46 | return response 47 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb/utils.lua: -------------------------------------------------------------------------------- 1 | local utils = {} 2 | 3 | local fiber = require('fiber') 4 | local json = require('json') 5 | 6 | function utils.now() 7 | return fiber.time64() 8 | end 9 | 10 | function utils.to_json(data) 11 | return json.encode(data) 12 | end 13 | 14 | function utils.try(f, catch_f) 15 | local status, exception = pcall(f) 16 | if not status and catch_f then 17 | catch_f(exception) 18 | end 19 | end 20 | 21 | function utils.map(f, coll) 22 | local res = {} 23 | for idx, x in pairs(coll) do 24 | res[idx] = f(x) 25 | end 26 | return res 27 | end 28 | 29 | return utils 30 | -------------------------------------------------------------------------------- /src/main/tarantool/xtdb/validator.lua: -------------------------------------------------------------------------------- 1 | local validator = {} 2 | 3 | local log = require('log') 4 | 5 | local config_default_values = {} 6 | 7 | local config_default_space_names = { 8 | tx_log = 'xtdb_tx_log', 9 | kv_store = 'xtdb_kv_store', 10 | document_store = 'xtdb_document_store', 11 | } 12 | 13 | function validator.is_nil(x) 14 | return x == nil 15 | end 16 | 17 | function validator.is_some(x) 18 | return x ~= nil 19 | end 20 | 21 | function validator.is_string(x) 22 | return type(x) == 'string' 23 | end 24 | 25 | function validator.not_blank_string(x) 26 | return validator.is_string(x) and x ~= '' 27 | end 28 | 29 | function validator.is_number(x) 30 | return type(x) == 'number' 31 | end 32 | 33 | function validator.is_pos(x) 34 | return validator.is_number(x) and x >= 0 35 | end 36 | 37 | function validator.is_table(x) 38 | return type(x) == 'table' 39 | end 40 | 41 | function validator.is_empty(x) 42 | return validator.is_nil(next(x)) 43 | end 44 | 45 | function validator.validate(config) 46 | if not validator.is_table(config) then 47 | config = {} 48 | log.warn('Config is invalid. The config must be a table. The default values will be applied.') 49 | end 50 | 51 | for param_name, default_value in pairs(config_default_values) do 52 | param_value = config[param_name] 53 | if validator.is_nil(param_value) or not validator.is_pos(param_value) then 54 | config[param_name] = default_value 55 | log.warn('Apply %s for %s', default_value, param_name) 56 | end 57 | end 58 | 59 | if not validator.is_table(config.spaces) then 60 | config.spaces = {} 61 | end 62 | 63 | for param_name, default_value in pairs(config_default_space_names) do 64 | if not (validator.is_table(config.spaces[param_name]) and validator.not_blank_string(config.spaces[param_name].name)) then 65 | config.spaces[param_name] = {} 66 | config.spaces[param_name].name = default_value 67 | end 68 | end 69 | 70 | return config 71 | 72 | end 73 | 74 | return validator 75 | -------------------------------------------------------------------------------- /src/test/clojure/xtdb/tarantool_test.clj: -------------------------------------------------------------------------------- 1 | (ns xtdb.tarantool-test 2 | (:require 3 | [clojure.test :refer [deftest testing is]] 4 | [xtdb.api :as xt] 5 | [xtdb.system :as system] 6 | [xtdb.tarantool :as sut] 7 | [xtdb.tx.event :as xte]) 8 | (:import 9 | (io.tarantool.driver.core 10 | RetryingTarantoolTupleClient) 11 | (java.util 12 | ArrayList) 13 | (java.util.concurrent 14 | ExecutionException) 15 | (java.util.function 16 | Function 17 | UnaryOperator))) 18 | 19 | 20 | ;; 21 | ;; Helper functions 22 | ;; 23 | 24 | (defn prep-tnt-client 25 | [overrides] 26 | (try 27 | (-> {:tnt-client (merge {:xtdb/module 'xtdb.tarantool/->tnt-client} overrides)} 28 | (system/prep-system) 29 | :tnt-client) 30 | (catch xtdb.IllegalArgumentException e 31 | (ex-message (ex-cause e))))) 32 | 33 | 34 | (defn start-tnt-client 35 | [overrides] 36 | (-> {:tnt-client (merge {:xtdb/module 'xtdb.tarantool/->tnt-client} overrides)} 37 | (system/prep-system) 38 | (system/start-system) 39 | :tnt-client)) 40 | 41 | 42 | (defn includes? 43 | [pattern x] 44 | (try 45 | (boolean (re-find pattern x)) 46 | (catch Exception _ 47 | false))) 48 | 49 | 50 | 51 | ;; 52 | ;; Tests 53 | ;; 54 | 55 | (deftest ^:unit default-exception-handler-test 56 | (testing "should be returned an instance of `java.util.function.Function`" 57 | (is (instance? Function (sut/default-exception-handler))))) 58 | 59 | 60 | 61 | (deftest ^:unit default-request-retry-policy-test 62 | (testing "should be returned an instance of `java.util.function.UnaryOperator`" 63 | (is (instance? UnaryOperator (sut/default-request-retry-policy {:delay 300}))))) 64 | 65 | 66 | 67 | (deftest to-inst-test 68 | (testing "should be returned an instance of `java.util.Date`" 69 | (is (= #inst"2021-12-03T01:26:09.506-00:00" (sut/to-inst 1638494769506799))))) 70 | 71 | 72 | 73 | (deftest prepare-fn-args-test 74 | (testing "should be returned an instance of `java.util.ArrayList`" 75 | (doseq [x [nil 1 1/2 "string" \c :keyword ::qualified-keyword 'symbol 'qualified-symbol '(1 2 3) [1 2 3] #{1 2 3} (ArrayList. [1 2 3])]] 76 | (is (instance? ArrayList (sut/prepare-fn-args x)))))) 77 | 78 | 79 | 80 | (deftest ^:unit ->tnt-client-specs-test 81 | (testing "should be failed by specs" 82 | 83 | (is (includes? #"Arg :username required" 84 | (prep-tnt-client {}))) 85 | 86 | (is (includes? #"Arg :username = 123[\s\S]+:xtdb\.tarantool\/username" 87 | (prep-tnt-client {:username 123}))) 88 | 89 | (is (includes? #"Arg :password required" 90 | (prep-tnt-client {:username "root"}))) 91 | 92 | (is (includes? #"Arg :password = 123[\s\S]+:xtdb\.tarantool\/password" 93 | (prep-tnt-client {:username "root" 94 | :password 123}))) 95 | 96 | (is (includes? #"Arg :host = 123[\s\S]+:xtdb\.tarantool\/host" 97 | (prep-tnt-client {:username "root" 98 | :password "root" 99 | :host 123}))) 100 | 101 | (is (includes? #"Arg :port = -123[\s\S]+:xtdb\.tarantool\/port" 102 | (prep-tnt-client {:username "root" 103 | :password "root" 104 | :host "127.0.0.1" 105 | :port -123}))) 106 | 107 | (is (includes? #"Arg :exception-handler = 123[\s\S]+:xtdb\.tarantool\/exception-handler" 108 | (prep-tnt-client {:username "root" 109 | :password "root" 110 | :exception-handler 123}))) 111 | 112 | (is (includes? #"Arg :request-retry-policy = 123[\s\S]+:xtdb\.tarantool\/request-retry-policy" 113 | (prep-tnt-client {:username "root" 114 | :password "root" 115 | :exception-handler (sut/default-exception-handler) 116 | :request-retry-policy 123}))) 117 | 118 | (is (includes? #"Arg :retries = -123[\s\S]+:xtdb\.tarantool\/retries" 119 | (prep-tnt-client {:username "root" 120 | :password "root" 121 | :exception (sut/default-exception-handler) 122 | :request-retry-policy (sut/default-request-retry-policy {:delay 500}) 123 | :retries -123})))) 124 | 125 | 126 | (testing "should be passed by specs" 127 | (is (nil? (prep-tnt-client {:username "root" :password "root"}))) 128 | 129 | (is (nil? (prep-tnt-client {:username "root" :password "root" :exception-handler (sut/default-exception-handler)}))) 130 | 131 | (is (nil? (prep-tnt-client {:username "root" 132 | :password "root" 133 | :exception-handler (sut/default-exception-handler) 134 | :request-retry-policy (sut/default-request-retry-policy {:delay 500})}))) 135 | 136 | (is (nil? (prep-tnt-client {:username "root" 137 | :password "root" 138 | :exception-handler (sut/default-exception-handler) 139 | :request-retry-policy (sut/default-request-retry-policy {:delay 500}) 140 | :retries 3}))))) 141 | 142 | 143 | 144 | (deftest ^:integration ->tnt-client-test 145 | (testing "should be throw an exception with a bad connection options" 146 | (let [tnt-client (start-tnt-client {:host "localhost" 147 | :port 3302 148 | :username "root" 149 | :password "root"})] 150 | (is (instance? RetryingTarantoolTupleClient tnt-client)) 151 | (is (thrown-with-msg? ExecutionException #"The client is not connected to Tarantool server" 152 | (sut/get-box-info tnt-client sut/default-complex-types-mapper))))) 153 | 154 | 155 | (testing "should be returned a successful response from the tarantool" 156 | (let [tnt-client (start-tnt-client {:username "root", :password "root"}) 157 | box-info (sut/get-box-info tnt-client sut/default-complex-types-mapper)] 158 | (is (instance? RetryingTarantoolTupleClient tnt-client)) 159 | (is (= "0.0.0.0:3301" (get box-info "listen"))) 160 | (is (includes? #"2.8.2" (get box-info "version"))) 161 | (sut/close tnt-client)))) 162 | 163 | 164 | 165 | (deftest ^:integration ->tx-log-test 166 | (let [tnt-client (start-tnt-client {:username "root", :password "root"}) 167 | mapper sut/default-complex-types-mapper 168 | tx-log {:tnt-client tnt-client, :mapper mapper} 169 | truncate #(sut/execute tnt-client mapper "xtdb.db.truncate")] 170 | (testing "should be returned the same tx-id" 171 | (truncate) 172 | (is (nil? (sut/latest-submitted-tx tx-log))) 173 | (let [tx @(sut/submit-tx tx-log [[::xt/put {:xt/id "hi2u", :user/name "zig"}]]) 174 | latest-tx (sut/latest-submitted-tx tx-log) 175 | txs (sut/open-tx-log* tx-log nil)] 176 | (is (every? seq [tx latest-tx txs])) 177 | (is (= #{::xt/tx-id} (->> latest-tx (keys) (set)))) 178 | (is (= #{::xt/tx-id ::xt/tx-time} (->> tx (keys) (set)))) 179 | (is (= #{::xt/tx-id ::xt/tx-time ::xte/tx-events} (->> txs (map keys) (flatten) (set)))) 180 | (is (= (::xt/tx-id tx) (::xt/tx-id latest-tx) (::xt/tx-id (last txs))))) 181 | (truncate) 182 | (Thread/sleep 5000) 183 | (sut/close tnt-client)))) 184 | 185 | 186 | 187 | (comment 188 | 189 | (def node 190 | (xt/start-node 191 | {::tnt-client {:xtdb/module `sut/->tnt-client 192 | :username "root" 193 | :password "root"} 194 | 195 | :xtdb/index-store {:xtdb/module 'xtdb.kv.index-store/->kv-index-store} 196 | :xtdb/tx-log {:xtdb/module `sut/->tx-log 197 | :tnt-client ::tnt-client}})) 198 | 199 | (sut/close node) 200 | 201 | 202 | (xt/submit-tx node [[::xt/put {:xt/id "hi2u", :user/name "zig"}]]) 203 | ;; => #:xtdb.api{:tx-id 4, :tx-time #inst"2021-12-03T22:00:34.561-00:00"} 204 | 205 | 206 | (xt/q (xt/db node) '{:find [e] 207 | :where [[e :user/name "zig"]]}) 208 | ;; => #{["hi2u"]} 209 | 210 | 211 | (xt/q (xt/db node) 212 | '{:find [(pull ?e [*])] 213 | :where [[?e :xt/id "hi2u"]]}) 214 | ;; => #{[{:user/name "zig", :xt/id "hi2u"}]} 215 | 216 | 217 | 218 | (def history (xt/entity-history (xt/db node) "hi2u" :desc {:with-docs? true})) 219 | ;;=> [#:xtdb.api{:tx-time #inst"2021-12-03T22:00:34.561-00:00", 220 | ;; :tx-id 4, 221 | ;; :valid-time #inst"2021-12-03T22:00:34.561-00:00", 222 | ;; :content-hash #xtdb/id"32f90c7020097d1232d14a9af395fc6aabd02ad7", 223 | ;; :doc {:user/name "zig", :xt/id "hi2u"}}] 224 | 225 | 226 | (->> (map ::xt/doc history) 227 | (filter #(= (get % :user/name) "zig"))) 228 | ;; => ({:user/name "zig", :xt/id "hi2u"}) 229 | 230 | ) 231 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:kaocha/fail-fast? false 3 | :kaocha/color? true 4 | :kaocha/reporter [kaocha.report/documentation] 5 | 6 | :kaocha.plugin.randomize/randomize? true 7 | 8 | :capture-output? true 9 | 10 | :plugins [:kaocha.plugin/capture-output 11 | :kaocha.plugin/cloverage 12 | :kaocha.plugin/filter 13 | :kaocha.plugin/hooks 14 | :kaocha.plugin/notifier 15 | :kaocha.plugin/print-invocations 16 | :kaocha.plugin/randomize 17 | :kaocha.plugin.alpha/info] 18 | 19 | :tests [{:id :unit 20 | :source-paths ["src/main/clojure"] 21 | :test-paths ["src/test/clojure"] 22 | :focus-meta [:unit]} 23 | {:id :integration 24 | :source-paths ["src/main/clojure"] 25 | :test-paths ["src/test/clojure"] 26 | :focus-meta [:integration]}] 27 | 28 | :cloverage/opts {:output "coverage" 29 | :ns-regex [] 30 | :ns-exclude-regex [] 31 | :fail-threshold 0 32 | :low-watermark 50 33 | :high-watermark 80 34 | :summary? true 35 | :text? false 36 | :emma-xml? false 37 | :html? true 38 | :nop? false 39 | :lcov? false 40 | :coveralls? false 41 | :codecov? true}} 42 | -------------------------------------------------------------------------------- /version.tmpl: -------------------------------------------------------------------------------- 1 | 0.1.{{build-number}} 2 | --------------------------------------------------------------------------------