├── .gitignore
├── README.md
├── boot.properties
├── build.boot
├── cnf
    └── logback.xml
└── src
    └── tailrecursion
        └── boot_datomic.clj


/.gitignore:
--------------------------------------------------------------------------------
 1 | ### BOOT #######################################################################
 2 | 
 3 | /.boot/
 4 | 
 5 | ### LEININGEN ##################################################################
 6 | 
 7 | .lein-*
 8 | 
 9 | ### MAVEN (include pom.xml in /base) ###########################################
10 | 
11 | *.jar
12 | *.war
13 | pom.xml
14 | pom.xml.asc
15 | 
16 | ### NREPL ######################################################################
17 | 
18 | .repl-*
19 | .nrepl-*
20 | 
21 | ### JAVA #######################################################################
22 | 
23 | /hs_err_pid*.log
24 | 
25 | ### OSX ########################################################################
26 | 
27 | .DS_Store
28 | 
29 | ### EMACS ######################################################################
30 | 
31 | [#]*[#]
32 | 
33 | ### VIM ########################################################################
34 | 
35 | *.swn
36 | *.swo
37 | *.swp
38 | 
39 | ### PROJECT ####################################################################
40 | 
41 | /datomic*
42 | /log/
43 | /tgt/
44 | /data/
45 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # boot datomic (wip)
  2 | run the datomic transactor in a pod. backup and restore the db using boot tasks.
  3 | 
  4 | deployment tasks forthcoming. sometime. eventually.
  5 | 
  6 | [](dependency)
  7 | ```clojure
  8 | [tailrecursion/boot-datomic "0.1.3"] ;; latest release
  9 | ```
 10 | [](/dependency)
 11 | 
 12 | ## configuration
 13 | 
 14 | install the transactor libary and its dependent libraries into your local maven repo.  ideally, someone at cognitect would notice this and decide it would be a great idea to distribute the transactor alongside the peer in the datomic repo. :)
 15 | 
 16 | ```bash
 17 | boot install-jars -f datomic-pro-0.9.5561.50/datomic-transactor-pro-0.9.5703.jar -f datomic-pro-0.9.5703/lib
 18 | ```
 19 | 
 20 | set the DATOMIC LICENSE KEY environment variable.
 21 | ```bash
 22 | export DATOMIC_LICENSE_KEY=<license-key>
 23 | ```
 24 | 
 25 | add boot-datomic and your preferred logging framework to the environment.
 26 | 
 27 | ## application
 28 | for the sake of expediency, here's an example of a `build.boot` file using this task:
 29 | ```clojure
 30 | (set-env!
 31 |   :resource-paths #{"src"}
 32 |   :source-paths   #{"ini" "rsc"}
 33 |   :dependencies   '[[com.datomic/datomic-pro         "0.9.5703" :exclusions [org.slf4j/slf4j-nop org.slf4j/slf4j-log4j12]]
 34 |                     [tailrecursion/boot-datomic      "0.1.2-SNAPSHOT"]
 35 |                     [ch.qos.logback/logback-classic  "1.2.3"] ]
 36 |   :repositories  [["clojars"       "https://clojars.org/repo/"]
 37 |                   ["maven-central" "https://repo1.maven.org/maven2/"]
 38 |                   ["datomic"       {:url      "https://my.datomic.com/repo"
 39 |                                     :username (System/getenv "DATOMIC_REPO_USERNAME")
 40 |                                     :password (System/getenv "DATOMIC_REPO_PASSWORD") }]
 41 |                   ["companyname"   {:url      "s3p://companyname.repository/snapshot" ;; transactor jar
 42 |                                     :username (System/getenv "COMPANY_AWS_ACCESS_KEY")
 43 |                                     :password (System/getenv "COMPANY_AWS_SECRET_KEY") }]])
 44 | 
 45 | (require
 46 |   '[tailrecursion.boot-datomic :refer [datomic backup restore]] )
 47 | 
 48 | (def datomic-data-readers
 49 |   {'base64 datomic.codec/base-64-literal
 50 |    'db/id  datomic.db/id-literal
 51 |    'db/fn  datomic.function/construct })
 52 | 
 53 | (def backup-uris
 54 |   {:production  "bak/production"
 55 |    :staging     "bak/staging"
 56 |    :development "bak/development" })
 57 | 
 58 | (def datomic-uris
 59 |   {:production  "datomic:ddb://us-east-1/foo/bar"
 60 |    :staging     "datomic:ddb://us-east-1/foo/bar"
 61 |    :development "datomic:dev://localhost:4334/bar" })
 62 | 
 63 | (defn backup-uri [environment]
 64 |   (.toString (.toURI (io/file (backup-uris environment)))) )
 65 | 
 66 | (defn datomic-uri [environment]
 67 |   (let [acc-key  (System/getenv "COMPANY_AWS_ACCESS_KEY")
 68 |         sec-key  (System/getenv "COMPANY_AWS_SECRET_KEY")
 69 |         rmt-env? (or (= environment :production) (= environment :staging))
 70 |         base-uri (datomic-uris environment) ]
 71 |     (assert acc-key "Missing environment variable COMPANY_AWS_ACCESS_KEY.")
 72 |     (assert sec-key "Missing environment variable COMPANY_AWS_SECRET_KEY.")
 73 |     (info "Connecting datomic to %s %s environment...\n" (if rmt-env? "remote" "local") (name environment))
 74 |     (str base-uri (if rmt-env? (str "?aws_access_key_id=" acc-key "&aws_secret_key=" sec-key))) ))
 75 | 
 76 | (deftask run
 77 |   []
 78 |   (comp (wait) (speak) (datomic :license-key (System/getenv "DATOMIC_LICENSE_KEY")) ))
 79 | 
 80 | (deftask backup
 81 |   [e environment ENV  kw "The application environment to backup."]
 82 |   (assert environment "Missing required environment argument.")
 83 |   (db/backup :from-db-uri (datomic-uri environment) :to-backup-uri (backup-uri environment)) )
 84 | 
 85 | (deftask restore
 86 |   [e environment ENV  kw "The application environment to restore."]
 87 |   (assert environment "Missing required environment argument.")
 88 |   (db/restore :from-backup-uri (backup-uri environment) :to-db-uri (datomic-uri environment)) )
 89 | 
 90 | (deftask develop
 91 |   [e environment ENV kw "The backed up environment to restore the local dev db from."]
 92 |   (let [init* #(db/restore :from-backup-uri %1 :to-db-uri %2)
 93 |         init  #(if % (init* (backup-uri %) (datomic-uri :development)) identity) ]
 94 |     (comp (wait) (speak) (datomic) (init environment)) ))
 95 | ```
 96 | 
 97 | ## license
 98 | 
 99 | copyright (c) jumblerg. all rights reserved.
100 | 
101 | distributed with clojure under the eclipse public license
102 | 


--------------------------------------------------------------------------------
/boot.properties:
--------------------------------------------------------------------------------
1 | #http://boot-clj.com
2 | #Sun Aug 05 10:21:19 EDT 2018
3 | BOOT_CLOJURE_NAME=org.clojure/clojure
4 | BOOT_CLOJURE_VERSION=1.9.0
5 | BOOT_VERSION=2.8.1
6 | 


--------------------------------------------------------------------------------
/build.boot:
--------------------------------------------------------------------------------
 1 | (set-env!
 2 |   :source-paths   #{"cnf"}
 3 |   :resource-paths #{"src"}
 4 |   :target-path    "tgt"
 5 |   :dependencies   '[[org.clojure/clojure                 "1.9.0"  :scope "provided"]
 6 |                     [ch.qos.logback/logback-classic      "1.2.3"  :scope "provided"]
 7 |                     [adzerk/bootlaces                    "0.1.13" :scope "test"]]
 8 |   :repositories    [["clojars"       "https://clojars.org/repo/"]
 9 |                     ["maven-central" "https://repo1.maven.org/maven2/"]
10 |                     ["datomic"       {:url      "https://my.datomic.com/repo"
11 |                                       :username (System/getenv "DATOMIC_REPO_USERNAME")
12 |                                       :password (System/getenv "DATOMIC_REPO_PASSWORD")}]])
13 | 
14 | (require
15 |   '[adzerk.bootlaces           :refer [bootlaces! build-jar push-release]]
16 |   '[tailrecursion.boot-datomic :refer :all])
17 | 
18 | (def +version+ "0.1.3")
19 | 
20 | (bootlaces! +version+)
21 | 
22 | (deftask run
23 |   []
24 |   (comp (wait) (speak) (datomic)))
25 | 
26 | (deftask develop []
27 |   (comp (watch) (speak) (build-jar)))
28 | 
29 | (deftask deploy []
30 |   (comp (build-jar) (push-release)))
31 | 
32 | (task-options!
33 |   pom     {:project     'tailrecursion/boot-datomic
34 |            :version     +version+
35 |            :description "Boot task to run the datomic transactor."
36 |            :url         "https://github.com/tailrecursion/boot-datomic"
37 |            :scm         {:url "https://github.com/tailrecursion/boot-datomic"}
38 |            :license     {"EPL" "http://www.eclipse.org/legal/epl-v10.html"}}
39 |   datomic {:license-key (System/getenv "DATOMIC_LICENSE_KEY")})
40 | 


--------------------------------------------------------------------------------
/cnf/logback.xml:
--------------------------------------------------------------------------------
 1 | <configuration>
 2 | 
 3 |   <!-- prevent per-message overhead for jul logging calls, e.g. Hornet -->
 4 |   <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
 5 |     <resetJUL>true</resetJUL>
 6 |   </contextListener>
 7 | 
 8 |   <appender name="MAIN" class="ch.qos.logback.core.rolling.RollingFileAppender">
 9 |     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
10 |       <fileNamePattern>${DATOMIC_LOG_DIR:-log}/%d{yyyy-MM-dd}.log</fileNamePattern>
11 |       <maxHistory>72</maxHistory> 
12 |     </rollingPolicy>
13 |     <prudent>true</prudent> <!-- multi jvm safe, slower -->
14 |     <encoder>
15 |       <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %-10contextName %logger{36} - %msg%n</pattern>
16 |     </encoder>
17 |   </appender>
18 | 
19 |   <!-- uncomment to log storage access -->
20 |   <!-- <logger name="datomic.kv-cluster" level="DEBUG"/> -->
21 | 
22 |   <!-- uncomment to log transactor heartbeat -->
23 |   <!-- <logger name="datomic.lifecycle" level="DEBUG"/> -->
24 | 
25 |   <!-- uncomment to log transactions (transactor side) -->
26 |   <!-- <logger name="datomic.transaction" level="DEBUG"/> -->
27 | 
28 |   <!-- uncomment to log transactions (peer side) -->
29 |   <!-- <logger name="datomic.peer" level="DEBUG"/> -->
30 | 
31 |   <!-- uncomment to log the transactor log -->
32 |   <!-- <logger name="datomic.log" level="DEBUG"/> -->
33 | 
34 |   <!-- uncomment to log peer connection to transactor -->
35 |   <!-- <logger name="datomic.connector" level="DEBUG"/> -->
36 | 
37 |   <!-- uncomment to log storage gc -->
38 |   <!-- <logger name="datomic.garbage" level="DEBUG"/> -->
39 | 
40 |   <!-- uncomment to log indexing jobs -->
41 |   <!-- <logger name="datomic.index" level="DEBUG"/> -->
42 | 
43 |   <!-- these namespsaces create a ton of log noise -->
44 |   <logger name="httpclient" level="INFO"/>
45 |   <logger name="org.apache.commons.httpclient" level="INFO"/>
46 |   <logger name="org.apache.http" level="INFO"/>
47 |   <logger name="org.jets3t" level="INFO"/>
48 |   <logger name="com.amazonaws" level="INFO"/>
49 |   <logger name="com.amazonaws.request" level="WARN"/>
50 |   <logger name="sun.rmi" level="INFO"/> 
51 |   <logger name="net.spy.memcached" level="INFO"/>
52 |   <logger name="com.couchbase.client" level="INFO"/>
53 |   <logger name="org.apache.zookeeper" level="INFO"/>
54 |   <logger name="com.ning.http.client.providers.netty" level="INFO"/>
55 |   <logger name="org.eclipse.jetty" level="INFO"/>
56 |   <logger name="org.hornetq.core.client.impl" level="INFO"/>
57 |   <logger name="org.apache.tomcat.jdbc.pool" level="INFO"/>
58 | 
59 |   <root level="info">
60 |     <appender-ref ref="MAIN"/>
61 |   </root>
62 | </configuration>


--------------------------------------------------------------------------------
/src/tailrecursion/boot_datomic.clj:
--------------------------------------------------------------------------------
  1 | (ns tailrecursion.boot-datomic
  2 |   {:boot/export-tasks true}
  3 |   (:require
  4 |     [clojure.java.io :as io]
  5 |     [boot.pod        :as pod]
  6 |     [boot.util       :as util]
  7 |     [boot.core       :as core :refer [deftask]]))
  8 | 
  9 | (deftask install-jars
 10 |   "Install the jars with maven pom files"
 11 |   [f files PATH #{str} "Paths to jar files and directories."]
 12 |   (let [{dirs true files false} (group-by #(.isDirectory %) (map io/file files))
 13 |         files                   (concat (mapcat file-seq dirs) files)]
 14 |     (core/with-pass-thru [fs]
 15 |       (doseq [jarpath (filter #(re-find #".jar$" %) (map #(.getPath %) files))]
 16 |         (util/info "Installing %s...\n" jarpath)
 17 |         (try
 18 |           (pod/with-call-worker
 19 |             (boot.aether/install ~(core/get-env) ~jarpath nil))
 20 |           (catch Exception e
 21 |             (util/info "No pom file in jar %s. Installation skipped.\n" jarpath)))))))
 22 | 
 23 | (def ^:private deps
 24 |   "Datomic transactor to load if none is provided via the project."
 25 |   (->> '[[com.amazonaws/aws-java-sdk-dynamodb "1.11.382"]
 26 |          [com.datomic/datomic-transactor-pro  "0.9.5703" :exclusions [org.slf4j/slf4j-nop jline-win/jline-win bsh/bsh]]]
 27 |        (remove pod/dependency-loaded?)
 28 |        (cons '[org.clojure/clojure "1.9.0"])
 29 |        (delay)))
 30 | 
 31 | (defn make-pod []
 32 |   (-> (core/get-env)
 33 |       (update :dependencies into (vec (seq @deps)))
 34 |       (pod/make-pod)
 35 |       (future)))
 36 | 
 37 | (deftask create-dynamodb-table
 38 |   "Create a new DynamoDB table for use by Datomic.
 39 | 
 40 |   For more information reference http://docs.datomic.com/storage.html."
 41 | 
 42 |   [n table-name NAME      str "Required name of the table"
 43 |    g region REGION        kw  "Required AWS region"
 44 |    r read-capacity WRITES int "Optional read capacity (100)"
 45 |    w write-capacity READS int "Optional write capacity (50)"]
 46 | 
 47 |    (let [pod  (make-pod)
 48 |          opts (into {:read-capacity 100 :write-capacity 50} *opts*)]
 49 |       (core/with-pre-wrap fileset
 50 |         (pod/with-call-in @pod
 51 |           (datomic.provisioning.aws/create-system-command ~opts))
 52 |         fileset)))
 53 | 
 54 | (deftask backup
 55 |   "Backup the database.
 56 | 
 57 |   The destination URI may refer to the local filesystem or an S3 bucket as
 58 |   shown below.
 59 | 
 60 |       file:/full/path/to/backup-directory
 61 |       s3://bucket/prefix
 62 | 
 63 |   Encryption is only supported on S3 using AWS' managed server side encryption
 64 |   keys (SSE-S3).
 65 | 
 66 |   For more information reference http://docs.datomic.com/backup.html."
 67 | 
 68 |   [s from-db-uri URI   str  "Required backup source"
 69 |    t to-backup-uri URI str  "Required backup target"
 70 |    e encryption        bool "Use AWS SSE-S3 encryption (false)"]
 71 | 
 72 |    (let [pod  (make-pod)
 73 |          opts (assoc *opts* :encryption (if encryption :sse))]
 74 |       (core/with-pre-wrap fileset
 75 |         (pod/with-call-in @pod
 76 |           (datomic.backup-cli/backup ~opts))
 77 |         fileset)))
 78 | 
 79 | (deftask list-backups
 80 |   "List the approximate points in time (t) of available backups made of the
 81 |   database.
 82 | 
 83 |   The source URI may refer to the local filesystem or an S3 bucket as shown
 84 |   below.
 85 | 
 86 |       file:/full/path/to/backup-directory
 87 |       s3://bucket/prefix
 88 | 
 89 |   For more information reference http://docs.datomic.com/backup.html."
 90 | 
 91 |   [s backup-uri URI str "Required backup source"]
 92 | 
 93 |    (let [pod  (make-pod)]
 94 |       (core/with-pre-wrap fileset
 95 |         (-> @pod
 96 |           (pod/with-call-in (datomic.backup-cli/list-backups ~*opts*))
 97 |           (println))
 98 |         fileset)))
 99 | 
100 | (deftask restore
101 |   "Restore the database.
102 | 
103 |   The source URI may refer to the local filesystem or an S3 bucket as shown
104 |   below.
105 | 
106 |       file:/full/path/to/backup-directory
107 |       s3://bucket/prefix
108 | 
109 |   This task restores the database to the most recent restoration point (t) by
110 |   default, but can optionally restore the database to another t value.
111 | 
112 |   For more information reference http://docs.datomic.com/backup.html."
113 | 
114 |   [s from-backup-uri URI str "Required restore source"
115 |    t to-db-uri URI       str "Required restore target"
116 |    m time T              int "Optional restoration point (most recent)"]
117 | 
118 |    (let [pod  (make-pod)
119 |          opts (clojure.set/rename-keys *opts* {:time :t})]
120 |       (core/with-pre-wrap fileset
121 |         (pod/with-call-in @pod
122 |           (datomic.backup-cli/restore ~opts))
123 |         fileset)))
124 | 
125 | (def ^:private transactor-defaults
126 |  {:protocol               "dev"
127 |   :host                   "localhost"
128 |   :port                   "4334"
129 |   :memory-index-max       "256m"
130 |   :memory-index-threshold "32m"
131 |   :object-cache-max       "128m"})
132 | 
133 | (deftask datomic
134 |   "Start the transactor.
135 | 
136 |   The settings are described in http://docs.datomic.com/system-properties.html.
137 | 
138 |   Memory settings default to the recommended settings for 1G of RAM appropriate for a laptop.
139 | 
140 |   For more information reference http://docs.datomic.com/capacity.html."
141 | 
142 |   [k license-key KEY              str  "License key (required)."
143 |    r protocol PROTOCOL            str  "Storage protocol (dev)."
144 |    u host HOST                    str  "Connection host (localhost)."
145 |    p port PORT                    str  "Connection port (4334)."
146 |    l log-dir PATH                 str  "The directory the logger will write to (/log)."
147 |    t memory-index-threshold BYTES str  "Threshold at which to start indexing (32m)."
148 |    m memory-index-max BYTES       str  "Maximum size of the memory index (256m)."
149 |    c object-cache-max BYTES       str  "Size of the object cache (128m)."
150 |    V volatile                     bool "Delete the data on each call (false)."]
151 | 
152 |    (let [dir      (core/cache-dir! ::data)
153 |          pod      (make-pod)
154 |          opts*    (assoc (dissoc *opts* :volatile) :data-dir (.getPath dir))
155 |          opts     (into transactor-defaults opts*)
156 |          message #(util/info "%s the Datomic transactor on port %s...\n" % (:port opts))
157 |          empty   #(when volatile (core/empty-dir! %)) ]
158 |       (core/cleanup
159 |         (message  "\nStopping")
160 |         (pod/with-call-in @pod
161 |           (datomic.api/shutdown true))
162 |         (empty dir) )
163 |       (core/with-pre-wrap fileset
164 |         (message  "Starting")
165 |         (empty dir)
166 |         (pod/with-call-in @pod
167 |           (datomic.transactor/run ~opts "datomic boot task options"))
168 |         (Thread/sleep 1000)
169 |         fileset)))
170 | 
171 | 
172 | 


--------------------------------------------------------------------------------