├── doc └── intro.md ├── nippy-example.edn ├── .gitignore ├── datomic-cloud-example.edn ├── datahike-file-example.edn ├── CHANGELOG.md ├── src └── wanderung │ ├── datahike.clj │ ├── cli.clj │ ├── datom.clj │ ├── db │ └── api.clj │ ├── datomic_cloud.clj │ └── core.clj ├── test └── wanderung │ ├── datomic_cloud_test.clj │ ├── datom_test.clj │ └── core_test.clj ├── template └── pom.xml ├── deps.edn ├── .circleci └── config.yml ├── resources └── testdatoms.edn ├── README.md ├── benchmark └── src │ └── wanderung │ └── benchmark │ └── core.clj └── LICENSE /doc/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to wanderung 2 | 3 | TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) 4 | -------------------------------------------------------------------------------- /nippy-example.edn: -------------------------------------------------------------------------------- 1 | ;; Store the raw datoms as nippy-encoded data in a file. 2 | {:wanderung/type :nippy 3 | :filename "backups/mybackup.nippy"} 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | profiles.clj 5 | *.jar 6 | *.class 7 | /.lein-* 8 | /.nrepl-port 9 | .hgignore 10 | .hg/ 11 | .idea 12 | /.clj-kondo 13 | /.lsp 14 | /.classpath 15 | /.project 16 | /.settings 17 | *.iml 18 | /.cpcache -------------------------------------------------------------------------------- /datomic-cloud-example.edn: -------------------------------------------------------------------------------- 1 | {:wanderung/type :datomic 2 | :name "your-database" 3 | :server-type :ion 4 | :region "eu-west-1" 5 | :system "your-system" 6 | :endpoint "http://entry.your-system.eu-west-1.datomic.net:8182/" 7 | :proxy-port 8182} 8 | -------------------------------------------------------------------------------- /datahike-file-example.edn: -------------------------------------------------------------------------------- 1 | {:wanderung/type :datahike 2 | :store {:backend :file 3 | :path "/tmp/dh"} 4 | ;; uncomment and adjust the following if the database does not exist 5 | ;; :schema-flexibility :write 6 | ;; :keep-history :true 7 | ;; :name "datahike-migration" 8 | } 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 3 | 4 | ## [Unreleased] 5 | ### Changed 6 | - Add a new arity to `make-widget-async` to provide a different widget shape. 7 | 8 | ## [0.1.1] - 2019-12-17 9 | ### Changed 10 | - Documentation on how to make the widgets. 11 | 12 | ### Removed 13 | - `make-widget-sync` - we're all async, all the time. 14 | 15 | ### Fixed 16 | - Fixed widget maker to keep working when daylight savings switches over. 17 | 18 | ## 0.1.0 - 2019-12-17 19 | ### Added 20 | - Files from the new template. 21 | - Widget maker public API - `make-widget-sync`. 22 | 23 | [Unreleased]: https://github.com/your-name/wanderung/compare/0.1.1...HEAD 24 | [0.1.1]: https://github.com/your-name/wanderung/compare/0.1.0...0.1.1 25 | -------------------------------------------------------------------------------- /src/wanderung/datahike.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.datahike 2 | (:require [datahike.api :as d])) 3 | 4 | (def find-tx-datoms 5 | '[:find ?tx ?inst 6 | :in $ ?bf 7 | :where 8 | [?tx :db/txInstant ?inst] 9 | [(< ?bf ?inst)]]) 10 | 11 | (def find-datoms-in-tx 12 | '[:find ?e ?a ?v ?t ?added 13 | :in $ ?t 14 | :where 15 | [?e ?a ?v ?t ?added] 16 | (not [?e :db/txInstant ?v ?t ?added])]) 17 | 18 | (defn extract-datahike-data 19 | "Given a Datahike connection extracts datoms from indices." 20 | [conn] 21 | (let [db (d/db conn) 22 | txs (sort-by first (d/q find-tx-datoms db (java.util.Date. 70))) 23 | query {:query find-datoms-in-tx 24 | :args [(d/history db)]}] 25 | (mapcat 26 | (fn [[tid tinst]] 27 | (->> (d/q (update-in query [:args] conj tid)) 28 | (sort-by first) 29 | (into [[tid :db/txInstant tinst tid true]]))) 30 | txs))) 31 | 32 | -------------------------------------------------------------------------------- /test/wanderung/datomic_cloud_test.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.datomic-cloud-test 2 | (:require [clojure.test :refer :all] 3 | [wanderung.datomic-cloud :as datomic-cloud])) 4 | 5 | (def sample-datom [119 :person/name "Mjao" nil true]) 6 | (def sample-datom-2 [119 :person/mother 120 nil true]) 7 | 8 | (def datom->list-form #'datomic-cloud/datom->list-form) 9 | 10 | (deftest datom->list-form-test 11 | (is (= [:db/add "tmp-119" :person/name "Mjao"] 12 | (datom->list-form {:ref-attribs #{} :tempids {}} 13 | sample-datom))) 14 | (is (= [:db/add 49 :person/name "Mjao"] 15 | (datom->list-form {:ref-attribs #{} 16 | :tempids {"tmp-119" 49 17 | "tmp-120" 50}} 18 | sample-datom))) 19 | (is (= [:db/add 17 :person/mother 1024] 20 | (datom->list-form {:ref-attribs #{:person/mother} 21 | :tempids {"tmp-119" 17 22 | "tmp-120" 1024}} 23 | sample-datom-2)))) 24 | -------------------------------------------------------------------------------- /template/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | io.lambdaforge 5 | wanderung 6 | jar 7 | wanderung 8 | A migration tool for Datahike from and to other databases. 9 | https://github.com/lambdaforge/wanderung 10 | 11 | 12 | Eclipse 13 | http://www.eclipse.org/legal/epl-v10.html 14 | 15 | 16 | 17 | scm:git:git@github.com:lambdaforge/wanderung.git 18 | scm:git:git@github.com/lambdaforge/wanderung.git 19 | https://github.com/lambdaforge/wanderung 20 | 21 | 22 | -------------------------------------------------------------------------------- /test/wanderung/datom_test.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.datom-test 2 | (:require [clojure.test :refer :all] 3 | [clojure.spec.alpha :as spec] 4 | [clojure.java.io :as io] 5 | [clojure.edn :as edn] 6 | [wanderung.datom :as datom])) 7 | 8 | (def testdata (-> "testdatoms.edn" 9 | io/resource 10 | slurp 11 | edn/read-string)) 12 | 13 | (deftest valid-tuples-test 14 | (is (spec/valid? :datom/tuples testdata))) 15 | 16 | (deftest entity-attribute-map-test 17 | (let [m (datom/datoms->entity-attribute-map testdata)] 18 | (-> m 19 | (get-in [17592186045426 :db/ident]) 20 | (= :green) 21 | is) 22 | (is (= #{:db.install/attribute :inv/color :inv/type :inv/size} 23 | (datom/ref-attrib-set m))))) 24 | 25 | (defn parameterized-datoms [eid-a eid-b tid] 26 | [[eid-a :person/name "Rudolf" tid true] 27 | [eid-b :person/name "Vera" tid true]]) 28 | 29 | (deftest normalized-equal-test 30 | (is (datom/similar-datoms? (parameterized-datoms 3 5 9) 31 | (parameterized-datoms 4 9 20))) 32 | (is (not (datom/similar-datoms? (parameterized-datoms 3 5 9) 33 | (assoc-in (parameterized-datoms 4 9 20) 34 | [1 2] 119))))) 35 | -------------------------------------------------------------------------------- /src/wanderung/cli.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.cli 2 | (:require [wanderung.core :as wc] 3 | [clojure.set :refer [rename-keys]] 4 | [environ.core :refer [env]]) 5 | (:import [clojure.lang IExceptionInfo])) 6 | 7 | (defn- run-it! [f] 8 | (try 9 | f 10 | (catch Throwable t 11 | (println (.getMessage t)) 12 | (when-not (instance? IExceptionInfo t) 13 | (.printStackTrace t)) 14 | (System/exit 1)) 15 | (finally 16 | (shutdown-agents)))) 17 | 18 | (defn migrate [opts] 19 | (let [merged-opts (-> (select-keys env [:wanderung-source :wanderung-target]) 20 | (rename-keys {:wanderung-source :source 21 | :wanderung-target :target}) 22 | (merge opts))] 23 | (println "Running with opts: " merged-opts) 24 | (run-it! (wc/migration merged-opts)))) 25 | 26 | (def migration migrate) 27 | 28 | (def m migrate) 29 | 30 | (defn help 31 | ([] 32 | (help {})) 33 | ([_] 34 | (try 35 | (println "WANDERUNG") 36 | (println "---------") 37 | (println "Run migrations with Datahike to and from various sources") 38 | (println "USAGE:") 39 | (println "clj -Twanderung [function] [function args]") 40 | (println "FUNCTIONS:") 41 | (println "---------") 42 | (println "migrate/migration/m :source SOURCE :target TARGET") 43 | (println "Description: Migrates from given source file to a target file. Source and target must be either file path or environment variable. Use WANDERUNG_SOURCE and WANDERUNG_TARGET environment variables if you don't want to use any input.") 44 | (println "Example: clj -Twanderung migrate :source '\"./source-cfg.edn\"' :target '\"target-cfg.edn\"'") 45 | (println "Example: clj -Twanderung m :source 'SOURCE_CFG' :target 'TARGET_CFG'") 46 | (println "Example: WANDERUNG_SOURCE=./source-cfg.edn WANDERUNG_TARGET=./target-cfg.edn clj -Twanderung migrate'") 47 | (println "---------") 48 | (println "help/h") 49 | (println "Description: Prints this lovely help.") 50 | (println "Example: clj -Twanderung help") 51 | (catch Throwable t 52 | (println (.getMessage t)) 53 | (when-not (instance? IExceptionInfo t) 54 | (.printStackTrace t)) 55 | (System/exit 1)) 56 | (finally 57 | (shutdown-agents))))) 58 | 59 | (def h help) 60 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | :deps {org.clojure/clojure {:mvn/version "1.10.3"} 3 | io.replikativ/konserve {:mvn/version "0.6.0-alpha3"} 4 | io.replikativ/datahike {:mvn/version "0.4.1488" :exclusions [io.replikativ/konserve]} 5 | com.datomic/client-cloud {:mvn/version "0.8.105"} 6 | com.cognitect/transit-clj {:mvn/version "0.8.313"} 7 | com.taoensso/nippy {:mvn/version "3.1.1"} 8 | environ/environ {:mvn/version "1.2.0"} 9 | org.clojure/tools.cli {:mvn/version "1.0.206"}} 10 | :tools/usage {:ns-default wanderung.cli} 11 | :aliases 12 | {:run-m {:main-opts ["-m" "wanderung.core"]} 13 | :dev {:extra-deps {org.clojure/test.check {:mvn/version "0.9.0"}} 14 | :ns-default wanderung.core} 15 | :test {:extra-paths ["test"] 16 | :extra-deps {com.datomic/dev-local {:mvn/version "1.0.242"} 17 | io.github.cognitect-labs/test-runner {:git/tag "v0.5.0" 18 | :git/sha "48c3c67"}}} 19 | :run-bench {:extra-paths ["benchmark/src"] 20 | :main-opts ["-m" "wanderung.benchmark.core"] 21 | :extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.0" 22 | :git/sha "48c3c67"}}} 23 | :format {:extra-deps {cljfmt/cljfmt {:mvn/version "0.8.0"}} 24 | :main-opts ["-m" "cljfmt.main" "check"]} 25 | 26 | :ffix {:extra-deps {cljfmt/cljfmt {:mvn/version "0.8.0"}} 27 | :main-opts ["-m" "cljfmt.main" "fix"]} 28 | :build {:deps {io.github.seancorfield/build-clj {:git/tag "v0.6.6" 29 | :git/sha "171d5f1"} 30 | borkdude/gh-release-artifact {:git/url "https://github.com/borkdude/gh-release-artifact" 31 | :sha "a83ee8da47d56a80b6380cbb6b4b9274048067bd"} 32 | babashka/babashka.curl {:mvn/version "0.1.1"} 33 | babashka/fs {:mvn/version "0.1.2"} 34 | cheshire/cheshire {:mvn/version "5.10.1"}} 35 | :ns-default build}} 36 | :mvn/repos {"s3mvn" {:url "s3://lambdaforge-blobs/maven/releases"}}} 37 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | github-cli: circleci/github-cli@1.0 5 | tools: replikativ/clj-tools@0 6 | 7 | jobs: 8 | setup: 9 | executor: tools/clojurecli 10 | steps: 11 | - restore_cache: 12 | keys: 13 | - source-{{ .Branch }}-{{ .Revision }} 14 | - source-{{ .Branch }} 15 | - source- 16 | - checkout 17 | - save_cache: 18 | key: source-{{ .Branch }}-{{ .Revision }} 19 | paths: 20 | - .git 21 | - restore_cache: 22 | keys: 23 | - deps-{{ checksum "deps.edn" }} 24 | - deps- 25 | - run: 26 | name: resolve deps 27 | command: clojure -P 28 | - save_cache: 29 | key: deps-{{ checksum "deps.edn" }} 30 | paths: 31 | - /home/circleci/.m2 32 | - persist_to_workspace: 33 | root: /home/circleci/ 34 | paths: 35 | - .m2 36 | - replikativ 37 | build: 38 | executor: tools/clojurecli 39 | steps: 40 | - attach_workspace: 41 | at: /home/circleci 42 | - run: 43 | name: clean 44 | command: clojure -Sthreads 1 -T:build clean 45 | - run: 46 | name: jar 47 | command: clojure -Sthreads 1 -T:build jar 48 | - persist_to_workspace: 49 | root: /home/circleci/ 50 | paths: 51 | - .m2 52 | - replikativ 53 | test: 54 | executor: tools/clojurecli 55 | steps: 56 | - attach_workspace: 57 | at: /home/circleci 58 | - run: 59 | name: test 60 | command: clojure -Sthreads 1 -T:build test 61 | no_output_timeout: 5m 62 | deploy: 63 | executor: tools/clojurecli 64 | steps: 65 | - attach_workspace: 66 | at: /home/circleci 67 | - run: 68 | name: deploy 69 | command: clojure -Sthreads 1 -T:build deploy 70 | 71 | workflows: 72 | build-test-and-deploy: 73 | jobs: 74 | - setup: 75 | context: clojars-deploy 76 | - build: 77 | context: clojars-deploy 78 | requires: 79 | - setup 80 | - tools/format: 81 | context: clojars-deploy 82 | requires: 83 | - setup 84 | - test: 85 | context: clojars-deploy 86 | requires: 87 | - build 88 | - deploy: 89 | context: clojars-deploy 90 | filters: 91 | branches: 92 | only: main 93 | requires: 94 | - tools/format 95 | - test 96 | -------------------------------------------------------------------------------- /resources/testdatoms.edn: -------------------------------------------------------------------------------- 1 | [[13194139534312 :db/txInstant #inst "2021-06-02T09:45:14.334-00:00" 13194139534312 true] [17592186045417 :db/ident :small 13194139534312 true] [17592186045418 :db/ident :medium 13194139534312 true] [17592186045419 :db/ident :large 13194139534312 true] [17592186045420 :db/ident :xlarge 13194139534312 true] [17592186045421 :db/ident :shirt 13194139534312 true] [17592186045422 :db/ident :pants 13194139534312 true] [17592186045423 :db/ident :dress 13194139534312 true] [17592186045424 :db/ident :hat 13194139534312 true] [17592186045425 :db/ident :red 13194139534312 true] [17592186045426 :db/ident :green 13194139534312 true] [17592186045427 :db/ident :blue 13194139534312 true] [17592186045428 :db/ident :yellow 13194139534312 true] [13194139534325 :db/txInstant #inst "2021-06-02T09:45:14.354-00:00" 13194139534325 true] [0 :db.install/attribute 72 13194139534325 true] [0 :db.install/attribute 73 13194139534325 true] [0 :db.install/attribute 74 13194139534325 true] [0 :db.install/attribute 75 13194139534325 true] [72 :db/unique :db.unique/identity 13194139534325 true] [72 :db/ident :inv/sku 13194139534325 true] [72 :db/cardinality :db.cardinality/one 13194139534325 true] [72 :db/valueType :db.type/string 13194139534325 true] [73 :db/cardinality :db.cardinality/one 13194139534325 true] [73 :db/valueType :db.type/ref 13194139534325 true] [73 :db/ident :inv/color 13194139534325 true] [74 :db/valueType :db.type/ref 13194139534325 true] [74 :db/cardinality :db.cardinality/one 13194139534325 true] [74 :db/ident :inv/size 13194139534325 true] [75 :db/cardinality :db.cardinality/one 13194139534325 true] [75 :db/valueType :db.type/ref 13194139534325 true] [75 :db/ident :inv/type 13194139534325 true] [13194139534326 :db/txInstant #inst "2021-06-02T09:45:14.382-00:00" 13194139534326 true] [17592186045431 :inv/color 17592186045425 13194139534326 true] [17592186045431 :inv/type 17592186045421 13194139534326 true] [17592186045431 :inv/sku "SKU-0" 13194139534326 true] [17592186045431 :inv/size 17592186045417 13194139534326 true] [17592186045432 :inv/size 17592186045417 13194139534326 true] [17592186045432 :inv/sku "SKU-1" 13194139534326 true] [17592186045432 :inv/color 17592186045425 13194139534326 true] [17592186045432 :inv/type 17592186045422 13194139534326 true] [17592186045433 :inv/type 17592186045423 13194139534326 true] [17592186045433 :inv/size 17592186045417 13194139534326 true] [17592186045433 :inv/color 17592186045425 13194139534326 true] [17592186045433 :inv/sku "SKU-2" 13194139534326 true] [17592186045434 :inv/sku "SKU-3" 13194139534326 true] [17592186045434 :inv/color 17592186045425 13194139534326 true] [17592186045434 :inv/size 17592186045417 13194139534326 true] [17592186045434 :inv/type 17592186045424 13194139534326 true] [17592186045435 :inv/sku "SKU-4" 13194139534326 true] [17592186045435 :inv/type 17592186045421 13194139534326 true] [17592186045435 :inv/color 17592186045425 13194139534326 true] [17592186045435 :inv/size 17592186045418 13194139534326 true]] 2 | -------------------------------------------------------------------------------- /src/wanderung/datom.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.datom 2 | (:require [clojure.spec.alpha :as spec])) 3 | 4 | (spec/def :datom/eid (spec/or :id number? :tempid string?)) 5 | (spec/def :datom/attribute keyword?) 6 | (spec/def :datom/value any?) 7 | (spec/def :datom/transaction-id :datom/eid) 8 | (spec/def :datom/add? boolean?) 9 | 10 | (spec/def :datom/tuple (spec/cat :eid :datom/eid 11 | :attribute :datom/attribute 12 | :value :datom/value 13 | :transaction-id :datom/transaction-id 14 | :add? :datom/add?)) 15 | 16 | (defn datom-eid [[eid _ _ _ _]] eid) 17 | (defn datom-attribute [[_ a _ _ _]] a) 18 | (defn datom-value [[_ _ v _ _]] v) 19 | (defn datom-tid [[_ _ _ tid _]] tid) 20 | (defn datom-add? [[_ _ _ _ add?]] add?) 21 | 22 | (spec/def :datom/tuples (spec/coll-of :datom/tuple)) 23 | 24 | (defn transaction-groups "Return a sequence of sequences of datoms with common transaction id" 25 | [datoms] 26 | (partition-by datom-tid datoms)) 27 | 28 | (defn datoms->entity-attribute-map "Given a list of datoms, create a map of all entities and their attributes after all list-forms have been applied. 29 | 30 | This implementation is only an approximation but good enough for figuring out which entities are part of the schema and used to define attributes whose values should be entity references." 31 | [datoms] 32 | (reduce 33 | (fn [dst [eid attr value tx add?]] 34 | (if add? 35 | (assoc-in dst [eid attr] value) 36 | (update dst eid dissoc attr))) 37 | {} 38 | datoms)) 39 | 40 | (defn datoms->entity-transaction-map 41 | "Build a map from every entity id to the first transaction id where it occurs" 42 | [datoms] 43 | (reduce (fn [dst [e a v t a?]] (update dst e #(or % t))) 44 | {} 45 | datoms)) 46 | 47 | (defn ref-attrib-set "Given an entity-attribute-map, get the set of attributes whose values are refs to other entities according to the schema" 48 | [entity-attribute-map] 49 | (into #{:db.install/attribute} 50 | (comp (map (fn [[k v]] (if (= :db.type/ref (:db/valueType v)) (:db/ident v)))) 51 | (filter some?)) 52 | entity-attribute-map)) 53 | 54 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 55 | ;;; 56 | ;;; Datoms similarity: Used for sanity checking 57 | ;;; 58 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 59 | 60 | (defn- inc-nil [x] 61 | (inc (or x 0))) 62 | 63 | (defn- build-eid-features 64 | "Constructs a map from each entity id to a feature value that is invariant of entity id assignment" 65 | [ref-attrib datoms] 66 | (let [] 67 | (reduce 68 | (fn [dst [tid transaction-group]] 69 | (reduce 70 | (fn [dst [eid attr value _ add?]] 71 | (let [step (fn [dst id what & data] (update-in dst [id [tid add? attr what data]] inc-nil))] 72 | (if (ref-attrib attr) 73 | (if (= eid value) 74 | (step dst eid :both) 75 | (-> dst 76 | (step eid :eid) 77 | (step value :value))) 78 | (step dst eid :eid value)))) 79 | dst 80 | transaction-group)) 81 | {} 82 | (map-indexed vector (transaction-groups datoms))))) 83 | 84 | (defn- datoms-feature 85 | "Compute a 'feature' value of all the datoms that is invariant of the entity id mapping" 86 | [datoms] 87 | (let [ref-attrib (-> datoms 88 | datoms->entity-attribute-map 89 | ref-attrib-set) 90 | eid-map (build-eid-features ref-attrib datoms)] 91 | (reduce 92 | (fn [dst [eid attr value tid add?]] 93 | (let [key [(eid-map eid) attr (if (ref-attrib attr) (eid-map value) value) (eid-map tid) add?]] 94 | (update dst key inc-nil))) 95 | {} 96 | datoms))) 97 | 98 | (defn similar-datoms? 99 | "Returns true if two sequences of datoms represent the same database. It generally returns false if they don't represent the same database, but can in rare cases return true. This function can nevertheless be used for sanity checking in order to detect common errors." 100 | [datoms-a datoms-b] 101 | (= (datoms-feature datoms-a) 102 | (datoms-feature datoms-b))) 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # wanderung 2 | 3 | Migration tool for Datahike from and to other databases. 4 | 5 | ## Usage 6 | 7 | Make sure your source and target databases exist. You can run the migration on the commandline using [clojure CLI](https://clojure.org/reference/deps_and_cli): 8 | 9 | ```bash 10 | clj -M:run-m -s datomic-cloud.edn -t datahike-file.edn 11 | ``` 12 | 13 | Use `clj -M:run-m -h` for further instructions. See the `*-example.edn` files for `dataomic-cloud`, `datahike-file` or `nippy` example configurations. 14 | 15 | Alternatively open your Clojure project, add `io.lambdaforge/wanderung` to your dependencies, and start a REPL: 16 | 17 | ```clojure 18 | (require '[wanderung.core :as w]) 19 | 20 | (def datomic-cfg {:wanderung/type :datomic 21 | :name "your-database" 22 | :server-type :ion 23 | :region "eu-west-1" 24 | :system "your-system" 25 | :endpoint "http://entry.your-system.eu-west-1.datomic.net:8182/" 26 | :proxy-port 8182}) 27 | 28 | (def datahike-cfg {:wanderung/type :datahike 29 | :store {:backend :file 30 | :path "/your-data-path"} 31 | :name "from-datomic" 32 | :schema-flexibility :write 33 | :keep-history? true}) 34 | ;; if the database doesn't exist, wanderung will create a Datahike database 35 | 36 | (w/migrate datomic-cfg datahike-cfg) 37 | ``` 38 | You can use `clj -T:build jar` to create a jar file and `clj -T:build install` to install the library in your local maven repository. 39 | Take a look at `build.clj` for further commands. 40 | 41 | ### CLI tools 42 | 43 | If you have Clojure [CLI tools](https://clojure.org/guides/deps_and_cli) installed you can install `wanderung` locally and use it as a commandline tool. 44 | 45 | Install it with: 46 | ```bash 47 | clj -Ttools install io.lambdaforge/wanderung '{:git/url "https://github.com/lambdaforge/wanderung" :git/tag "v0.2.67"}' :as wanderung 48 | ``` 49 | 50 | Make sure it is installed with: 51 | ```bash 52 | clj -Ttools list 53 | ``` 54 | 55 | Run it with: 56 | ```bash 57 | clj -Twanderung migration :source '"./source-cfg.edn"' :target '"./target-cfg.edn"' 58 | ``` 59 | 60 | or with environment variables either inside `:source` and `:target` with your own naming: 61 | ```bash 62 | MY_SOURCE_CFG=./source-cfg.edn 63 | MY_TARGET_CFG=./target-cfg.edn 64 | clj -Twanderung m :source 'MY_SOURCE_CFG' :target 'MY_TARGET_CFG' 65 | ``` 66 | 67 | or with global environment variables defined by wanderung: 68 | ```bash 69 | WANDERUNG_SOURCE=./source-cfg.edn WANDERUNG_TARGET=./target-cfg.edn clj -Twanderung migrate 70 | ``` 71 | 72 | Show help with: 73 | ```bash 74 | clj -Twanderung help 75 | ``` 76 | 77 | Uninstall it with: 78 | ```bash 79 | clj -Ttools remove :tool wanderung 80 | ``` 81 | 82 | 83 | ## Tests 84 | 85 | Before using Wanderung for performing a migration, you may wish to run tests that to check that Wanderung works correctly. In order to do so, you need to perform the following steps: 86 | 87 | 1. Install [Datomic dev-local](https://docs.datomic.com/cloud/dev-local.html). 88 | 2. Run the tests by calling `clj -T:build test`. With `clj -T:build clean` you can clean up your local build files. 89 | 90 | ## Contributors 91 | - @perweij 92 | - @vlaaad 93 | - @jonasseglare 94 | 95 | ## Deprecation notice 96 | 97 | Starting from version `0.2.0` wanderung does not support leiningen as build tool anymore. Please adjust accordingly in your project when using wanderung from the commandline. 98 | 99 | ## License 100 | 101 | Copyright © 2020-2022 lambdaforge UG (haftungsbeschränkt) & Contributors 102 | 103 | This program and the accompanying materials are made available under the 104 | terms of the Eclipse Public License 2.0 which is available at 105 | http://www.eclipse.org/legal/epl-2.0. 106 | 107 | This Source Code may also be made available under the following Secondary 108 | Licenses when the conditions for such availability set forth in the Eclipse 109 | Public License, v. 2.0 are satisfied: GNU General Public License as published by 110 | the Free Software Foundation, either version 2 of the License, or (at your 111 | option) any later version, with the GNU Classpath Exception which is available 112 | at https://www.gnu.org/software/classpath/license.html. 113 | -------------------------------------------------------------------------------- /src/wanderung/db/api.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.db.api 2 | (:refer-clojure :exclude [sync]) 3 | (:require [datomic.client.api :as dc] 4 | [datahike.api :as dh])) 5 | 6 | (defn not-supported [] 7 | (throw (ex-info "Not supported yet." {:error/cause :not-supported}))) 8 | 9 | (defprotocol Historian 10 | (db-stats [db]) 11 | (history [db]) 12 | (as-of [db time-point]) 13 | (since [db time-point]) 14 | (with [db arg-map])) 15 | 16 | (defprotocol Searcher 17 | (datoms [db arg-map]) 18 | (pull [db arg-map] [db selector eid]) 19 | (index-range [db arg-map])) 20 | 21 | (defrecord DatahikeDb [state] 22 | Historian 23 | (db-stats [db] (not-supported)) 24 | (history [db] (map->DatahikeDb {:state (dh/history state)})) 25 | (as-of [db time-point] (map->DatahikeDb {:state (dh/as-of state time-point)})) 26 | (since [db time-point] (map->DatahikeDb {:state (dh/since state time-point)})) 27 | (with [db {:keys [tx-data]}] 28 | (dh/with state tx-data)) 29 | 30 | Searcher 31 | (datoms [db {:keys [index components]}] 32 | (dh/datoms state index components)) 33 | (pull [db {:keys [selector eid]}] 34 | (dh/pull state selector eid)) 35 | (pull [db selector eid] 36 | (dh/pull state selector eid)) 37 | (index-range [db arg-map] (not-supported))) 38 | 39 | (defrecord DatomicDb [state] 40 | Historian 41 | (db-stats [db] (not-supported)) 42 | (history [db] (map->DatomicDb {:state (dc/history state)})) 43 | (as-of [db time-point] (map->DatomicDb {:state (dc/as-of state time-point)})) 44 | (since [db time-point] (map->DatomicDb {:state (dc/since state time-point)})) 45 | (with [db arg-map] (dc/with state arg-map)) 46 | 47 | Searcher 48 | (datoms [db arg-map] (dc/datoms db arg-map)) 49 | (pull [db arg-map] (dc/pull state arg-map)) 50 | (pull [db selector eid] (dc/pull state selector eid)) 51 | (index-range [db arg-map] (not-supported))) 52 | 53 | (defprotocol Transactor 54 | (transact [conn arg-map]) 55 | (release [conn]) 56 | (db [conn]) 57 | (with-db [conn]) 58 | (sync [conn time-point]) 59 | (tx-range [conn arg-map])) 60 | 61 | (defrecord DatahikeConnection [state] 62 | Transactor 63 | (transact [conn arg-map] (dh/transact! state arg-map)) 64 | (release [conn] (dh/release state)) 65 | (db [conn] (map->DatahikeDb {:state @state})) 66 | (with-db [conn] @state) 67 | (sync [conn time-point] (not-supported)) 68 | (tx-range [conn arg-map] (not-supported))) 69 | 70 | (defrecord DatomicConnection [state] 71 | Transactor 72 | (transact [conn arg-map] (dc/transact state arg-map)) 73 | (release [conn]) 74 | (db [conn] (map->DatomicDb {:state (dc/db state)})) 75 | (with-db [conn] (dc/with-db state)) 76 | (sync [conn time-point] (not-supported)) 77 | (tx-range [conn arg-map] (not-supported))) 78 | 79 | (defprotocol Connector 80 | (connect [client arg-map])) 81 | 82 | (defprotocol Creator 83 | (administer-system [client]) 84 | (list-databases [client arg-map]) 85 | (create-database [client arg-map]) 86 | (delete-database [client arg-map])) 87 | 88 | (defrecord DatahikeClient [state] 89 | Connector 90 | (connect [client arg-map] 91 | (map->DatahikeConnection {:state (dh/connect state)})) 92 | 93 | Creator 94 | (administer-system [client] (not-supported)) 95 | (list-databases [client arg-map] (not-supported)) 96 | (create-database [client arg-map] 97 | (dh/create-database arg-map)) 98 | (delete-database [client arg-map] 99 | (dh/delete-database arg-map))) 100 | 101 | (defrecord DatomicClient [state] 102 | Connector 103 | (connect [client arg-map] 104 | (map->DatomicConnection {:state (dc/connect state arg-map)})) 105 | 106 | Creator 107 | (administer-system [client] (not-supported)) 108 | (list-databases [client arg-map] (not-supported)) 109 | (create-database [client arg-map] 110 | (dc/create-database state arg-map)) 111 | (delete-database [client arg-map] 112 | (dc/delete-database state arg-map))) 113 | 114 | (defn datahike-client [config] 115 | (map->DatahikeClient {:state config})) 116 | 117 | (defn datomic-client [config] 118 | (map->DatomicClient {:state (dc/client config)})) 119 | 120 | (defmulti -q-map (fn [{:keys [args] :as arg-map}] (-> args first class))) 121 | 122 | (defmethod -q-map DatahikeDb [arg-map] 123 | (dh/q (update-in arg-map [:args] (fn [old] (into [(-> old first :state)] (rest old)))))) 124 | 125 | (defmethod -q-map DatomicDb [arg-map] 126 | (dc/q (update-in arg-map [:args] (fn [old] (into [(-> old first :state)] (rest old)))))) 127 | 128 | (defmulti -q (fn [query & args] (-> args first class))) 129 | 130 | (defmethod -q DatahikeDb [query & args] 131 | (apply dh/q query (-> args first :state) (rest args))) 132 | 133 | (defmethod -q DatomicDb [query & args] 134 | (apply dc/q query (-> args first :state) (rest args))) 135 | 136 | (defn q 137 | ([arg-map] (-q-map arg-map)) 138 | ([query & args] (apply -q query args))) 139 | -------------------------------------------------------------------------------- /src/wanderung/datomic_cloud.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.datomic-cloud 2 | (:require [datomic.client.api :as d] 3 | [wanderung.datom :as datom] 4 | [clojure.spec.alpha :as spec])) 5 | 6 | (defn- temp-id [eid] 7 | (str "tmp-" eid)) 8 | 9 | (defn- map-value-ref 10 | "Maps the value part of a datom to either a temp id or a true entity id" 11 | [transaction-context value] 12 | (let [{:keys [tempids datom eid-tid-map]} transaction-context] 13 | (if (keyword? value) 14 | value 15 | (or (get tempids (temp-id value)) 16 | (let [this-tid (datom/datom-tid datom) 17 | value-tid (get eid-tid-map value)] 18 | (assert (number? this-tid)) 19 | (assert (number? value-tid)) 20 | (if (= this-tid value-tid) 21 | (temp-id value) 22 | (throw (ex-info "Failed to map value ref" 23 | {:value value 24 | :tempids tempids 25 | :datom datom 26 | :referred-entity (get (:entity-map transaction-context) value)})))))))) 27 | 28 | (def transaction-temp-id "datomic.tx") 29 | 30 | (defn- datom->list-form 31 | "Transforms a datom to list-form to be part of a transaction. Entity ids will be mapped." 32 | [transaction-context [eid attrib value tid add? :as datom]] 33 | (let [tempids (:tempids transaction-context) 34 | transaction-context (assoc transaction-context :datom datom) 35 | ref-attribs (:ref-attribs transaction-context) 36 | _ (assert ref-attribs) 37 | temp-eid (temp-id eid)] 38 | [;; Whether to add or retract 39 | (if add? :db/add :db/retract) 40 | 41 | ;; The entity id: either it is a temp id for a new entity, 42 | ;; or a true entity id. In the special case that the entity id is *this* transaction, 43 | ;; the special tempid "datomic.tx" is used. 44 | (if (= eid tid) ;; 45 | transaction-temp-id 46 | (get tempids temp-eid temp-eid)) 47 | 48 | ;; The attribute 49 | attrib 50 | 51 | ;; The value: If it is a reference to another entity, 52 | ;; the correct entity id needs to be figured out. 53 | (if (ref-attribs attrib) 54 | (if (= value tid) 55 | transaction-temp-id 56 | (map-value-ref transaction-context value)) 57 | value)])) 58 | 59 | (defn- tx-data-from-datoms [transaction-context datoms] 60 | "Map datoms to list-forms to be the tx-data of a transaction" 61 | (into [] 62 | (comp (remove #(#{:db.install/attribute} (datom/datom-attribute %))) 63 | (map (partial datom->list-form transaction-context))) 64 | datoms)) 65 | 66 | (defn- perform-transaction 67 | "Perform a transaction with the datoms and return an updated transaction-context" 68 | [conn transaction-context datoms] 69 | (let [tid (-> datoms first datom/datom-tid) 70 | tx-data (tx-data-from-datoms transaction-context datoms) 71 | result (d/transact 72 | conn 73 | {:tx-data tx-data}) 74 | new-tempids (:tempids result)] 75 | (update transaction-context 76 | :tempids 77 | #(-> % 78 | (merge new-tempids) 79 | (assoc (temp-id tid) (get new-tempids transaction-temp-id)))))) 80 | 81 | (defn transact-datoms-to-datomic "Transact datoms to datomic. This function does the opposite of what the function `extract-datomic-cloud-data` does." 82 | [conn datoms] 83 | {:pre [(spec/valid? :datom/tuples datoms)]} 84 | (let [entity-map (datom/datoms->entity-attribute-map datoms) 85 | ref-attribs (datom/ref-attrib-set entity-map) 86 | transaction-groups (datom/transaction-groups datoms)] 87 | (reduce (partial perform-transaction conn) 88 | {:ref-attribs ref-attribs 89 | :entity-map entity-map 90 | :eid-tid-map (datom/datoms->entity-transaction-map datoms) 91 | :datom nil 92 | :tempids {}} 93 | transaction-groups))) 94 | 95 | (defn create-schema-mapping [conn] 96 | (let [db (d/db conn) 97 | query '[:find ?e ?ident 98 | :where 99 | [?e :db/ident ?ident]]] 100 | (into {} (d/q query db)))) 101 | 102 | (defn extract-datomic-cloud-data 103 | "Extracts all transactions from Datomic with keyword attributes given a Datomic connection. 104 | Internally Datomic uses the first transactions to initialize the system schema and identifiers 105 | which are Datomic specific and not relevant for import. 106 | Currently, it takes 5 transactions, so the 6th is the first user specific one." 107 | [conn] 108 | (let [system-attributes #{:db.install/valueType :db/valueType :db/cardinality :db/unique} 109 | start-tx 6 ;; first user transaction 110 | schema-mapping (create-schema-mapping conn) 111 | map-db-ident (map (fn [[e a v tx added]] 112 | (let [new-a (schema-mapping a)] 113 | (assert new-a) 114 | [e new-a (if (system-attributes new-a) 115 | (schema-mapping v) 116 | v) tx added]))) 117 | data-extract (mapcat (fn [{:keys [data]}] 118 | (into [] map-db-ident data))) 119 | tx-data (d/tx-range conn {:start start-tx :limit -1})] 120 | (into [] data-extract tx-data))) -------------------------------------------------------------------------------- /src/wanderung/core.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.core 2 | (:require [datahike.api :as d] 3 | [datomic.client.api :as dt] 4 | [wanderung.datomic-cloud :as wdc] 5 | [wanderung.datahike :as wd] 6 | [wanderung.datom :as datom] 7 | [clojure.tools.cli :refer [parse-opts]] 8 | [clojure.java.io :as io] 9 | [taoensso.nippy :as nippy]) 10 | (:import [clojure.lang IExceptionInfo])) 11 | 12 | ;;;------- Basic datoms interface ------- 13 | 14 | ;; Two elementary methods for moving datoms to and from a database. 15 | (defmulti datoms-from-storage (fn [config] (:wanderung/type config))) 16 | (defmulti datoms-to-storage (fn [config datoms] (:wanderung/type config))) 17 | 18 | ;;; Datomic 19 | (defn datomic-connect [datomic-config] 20 | (println "➜ Connecting to Datomic...") 21 | (let [result (dt/connect (dt/client (dissoc datomic-config :name)) 22 | {:db-name (:name datomic-config)})] 23 | (println " ✓ Done") 24 | result)) 25 | 26 | (defmethod datoms-from-storage :datomic [storage] 27 | (-> storage 28 | datomic-connect 29 | wdc/extract-datomic-cloud-data)) 30 | 31 | (defmethod datoms-to-storage :datomic [storage datoms] 32 | (wdc/transact-datoms-to-datomic (datomic-connect storage) datoms)) 33 | 34 | ;;; Nippy 35 | (defmethod datoms-from-storage :nippy [storage] 36 | (nippy/thaw-from-file (:filename storage))) 37 | 38 | (defmethod datoms-to-storage :nippy [storage datoms] 39 | (let [filename (:filename storage)] 40 | (io/make-parents filename) 41 | (nippy/freeze-to-file filename datoms))) 42 | 43 | ;;; Datahike 44 | (defn datahike-maybe-create-and-connect [config] 45 | (when (not (d/database-exists? config)) 46 | (println "➜ Datahike database does not exist.") 47 | (println "➜ Creating database...") 48 | (d/create-database config) 49 | (println " ✓ Done")) 50 | (println "➜ Connecting to Datahike...") 51 | (let [result (d/connect config)] 52 | (println " ✓ Done") 53 | result)) 54 | 55 | (defmethod datoms-from-storage :datahike [storage] 56 | (-> storage 57 | d/connect 58 | wd/extract-datahike-data)) 59 | 60 | (defmethod datoms-to-storage :datahike [storage datoms] 61 | @(d/load-entities (datahike-maybe-create-and-connect storage) datoms) 62 | true) 63 | 64 | ;;;------- Migrations ------- 65 | 66 | (defmulti migrate (fn [source-configuration target-configuration] 67 | (mapv :wanderung/type [source-configuration 68 | target-configuration]))) 69 | 70 | #_(defmethod migrate [:datahike :datahike] [src-cfg tgt-cfg] 71 | ... optimized implementation for specific migration goes here ...) 72 | 73 | (defmethod migrate :default [src tgt] 74 | (->> (datoms-from-storage src) (datoms-to-storage tgt))) 75 | 76 | ;;;------- CLI ------- 77 | 78 | (defn multimethod-for-dispatch-value? [method x] 79 | (-> (methods method) keys set (contains? x))) 80 | 81 | (def cli-options 82 | [["-s" "--source SOURCE" "Source EDN configuration file" 83 | :parse-fn identity 84 | :validate [#(.exists (io/file %)) "Source configuration does not exist."]] 85 | ["-t" "--target TARGET" "Target EDN configuration file" 86 | :parse-fn identity 87 | :validate [#(.exists (io/file %)) "Target configuration does not exist."]] 88 | ["-h" "--help"] 89 | ["-c" "--check"]]) 90 | 91 | (defn load-config [filename] 92 | (-> filename 93 | slurp 94 | read-string)) 95 | 96 | (defn migration [{:keys [source target check]}] 97 | (let [read-cfg (fn [cfg] (when (some? cfg) 98 | (if-let [path-from-env (some? (System/getenv cfg))] 99 | path-from-env 100 | (when (.exists (io/file cfg)) 101 | cfg))))] 102 | (if-let [source (read-cfg source)] 103 | (if-let [target (read-cfg target)] 104 | (let [src-cfg (load-config source) 105 | tgt-cfg (load-config target) 106 | src-type (:wanderung/type src-cfg) 107 | tgt-type (:wanderung/type tgt-cfg)] 108 | (cond 109 | (not (multimethod-for-dispatch-value? datoms-from-storage src-type)) 110 | (println "Cannot use" src-type "as source database.") 111 | 112 | (not (multimethod-for-dispatch-value? datoms-to-storage tgt-type)) 113 | (println "Cannot use" tgt-type "as target database.") 114 | 115 | (:wanderung/read-only? tgt-cfg) 116 | (println "Cannot migrate to read-only database.") 117 | 118 | :else (do 119 | (println "➜ Start migrating from" src-type "to" tgt-type "...") 120 | (migrate src-cfg tgt-cfg) 121 | (println " ✓ Done") 122 | (when check 123 | (if (multimethod-for-dispatch-value? datoms-from-storage tgt-type) 124 | (do 125 | (println "➜ Comparing datoms between source and target...") 126 | (if (datom/similar-datoms? (datoms-from-storage src-cfg) 127 | (datoms-from-storage tgt-cfg)) 128 | (println " ✓ Success: Datoms look the same.") 129 | (println "ERROR: The datoms differ between source and target."))) 130 | (println "ERROR: The target does not support reading datoms")))))) 131 | (println "ERORR: Invalid target configuration. Must be either file path or environment variable.")) 132 | (println "ERORR: Invalid source configuration. Must be either file path or environment variable.")))) 133 | 134 | (defn -main [& args] 135 | (let [{options :options 136 | summary :summary 137 | errors :errors} (parse-opts args cli-options)] 138 | (if errors 139 | (->> errors (map println) doall) 140 | (if (:help options) 141 | (do 142 | (println "Run migrations to datahike from various sources") 143 | (println "USAGE:") 144 | (println summary)) 145 | (try 146 | (migration options) 147 | (catch Throwable t 148 | (println (.getMessage t)) 149 | (when-not (instance? IExceptionInfo t) 150 | (.printStackTrace t)) 151 | (System/exit 1)) 152 | (finally 153 | (shutdown-agents))))))) 154 | -------------------------------------------------------------------------------- /benchmark/src/wanderung/benchmark/core.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.benchmark.core 2 | (:require [wanderung.core :as w] 3 | [taoensso.timbre :as log] 4 | [clojure.tools.cli :refer [parse-opts]] 5 | [clojure.java.io :as io] 6 | [datahike.api :as d]) 7 | (:import [java.util Date]) 8 | (:gen-class)) 9 | 10 | (def valid-chars 11 | (map char (concat (range 48 58) 12 | (range 66 91) 13 | (range 97 123)))) 14 | 15 | (defn random-char [] 16 | (nth valid-chars (rand (count valid-chars)))) 17 | 18 | (defn random-str [length] 19 | (apply str (take length (repeatedly random-char)))) 20 | 21 | (defmacro timed 22 | "Evaluates expr. Returns the value of expr and the time in a map." 23 | [expr] 24 | `(let [start# (. System (nanoTime)) 25 | ret# ~expr] 26 | {:res ret# :t (/ (double (- (. System (nanoTime)) start#)) 1000000.0)})) 27 | 28 | (def cli-options 29 | [["-p" "--sample SAMPLE" "Sample size" 30 | :default 5 31 | :parse-fn (fn [x] (Long/parseLong x)) 32 | :validate [#(<= % 1000) "Too many samples. Please use a sample size between 0 and 1000"]] 33 | ["-n" "--tx-count TX_COUNT" "transaction count" 34 | :default 1000 35 | :parse-fn (fn [x] (Long/parseLong x)) 36 | :validate [#(<= % 1000000) "Too many Transactions. Please use a sample size between 0 and 1000000"]] 37 | ["-u" "--upsert-count UPSERT_COUNT" "upsert count" 38 | :default 1 39 | :parse-fn (fn [x] (Long/parseLong x)) 40 | :validate [#(<= 1 % 100) "Wrong upsert count. Please use an upsert size between 1 and 100"]] 41 | ["-e" "--entities-count ENTITIES_COUNT" "entities count" 42 | :default 1 43 | :parse-fn (fn [x] (Long/parseLong x)) 44 | :validate [#(<= 1 % 10000) "Too many entities. Please use an entity size between 1 and 10000"]] 45 | ["-s" "--source SOURCE" "Source EDN configuration file" 46 | :parse-fn identity 47 | :validate [#(.exists (io/file %)) "Source configuration does not exist."]] 48 | ["-o" "--output OUTPUT" "Output file. Appends to existing files." 49 | :parse-fn identity] 50 | ["-t" "--target TARGET" "Target EDN configuration file" 51 | :parse-fn identity 52 | :validate [#(.exists (io/file %)) "Target configuration does not exist."]] 53 | ["-b" "--backend BACKEND" "Index backend" 54 | :parse-fn keyword 55 | :default :file 56 | :validate [#(#{:file :mem} %) "Invalid index type."]] 57 | ["-i" "--index INDEX" "index type" 58 | :parse-fn keyword 59 | :default :hitchhiker-tree 60 | :validate [#(#{:hitchhiker-tree :persistent-set} %) "Invalid index type."]] 61 | ["-h" "--help"] 62 | ["-r" "--refs"]]) 63 | 64 | (defn init-source [source tx-count upsert-count entities-count] 65 | (d/delete-database source) 66 | (d/create-database source) 67 | (let [schema [{:db/ident :issue/id 68 | :db/cardinality :db.cardinality/one 69 | :db/valueType :db.type/string}] 70 | conn (d/connect source) 71 | _ (d/transact conn schema) 72 | upsert-tx (/ (* tx-count entities-count) upsert-count) 73 | entities-range (range entities-count)] 74 | (doseq [n (range tx-count)] 75 | (let [tx-data (mapv (fn [i] {:db/id (+ 1000 (mod (+ (* n entities-count) i) upsert-tx)) 76 | :issue/id (str "i" (+ (* n entities-count) i))}) entities-range)] 77 | (d/transact conn {:tx-data tx-data}))))) 78 | 79 | (defn avg-variance [results] 80 | (let [n (count results) 81 | avg (/ (reduce (fn [result {:keys [t]}] (+ result t)) 0.0 results) n) 82 | variance (/ (reduce (fn [result {:keys [t]}] (+ result (Math/pow (- t avg) 2.0))) 0.0 results) n)] 83 | {:avg avg 84 | :variance variance})) 85 | 86 | (defn print-results 87 | ([results] 88 | (print-results results nil)) 89 | ([results output-file] 90 | (if (some? output-file) 91 | (spit output-file results :append true) 92 | (println results)))) 93 | 94 | (defn -main [& args] 95 | (log/set-level! :info) 96 | (let [{options :options 97 | summary :summary 98 | errors :errors} (parse-opts args cli-options)] 99 | (if errors 100 | (->> errors (map println) doall) 101 | (if (:help options) 102 | (do 103 | (println "Run migrations to datahike from various sources") 104 | (println "USAGE:") 105 | (println summary)) 106 | (let [{:keys [sample tx-count upsert-count entities-count source target output index backend]} options] 107 | (println "Used options:" options) 108 | (if (some? source) 109 | (if (some? target) 110 | (let [{:keys [t]} (timed (w/migrate source target))] 111 | (print-results {:t t} output)) 112 | (let [target-name (str "wanderung_" (random-str 8)) 113 | target (-> source 114 | (assoc-in [:store :path] (str (System/getProperty "java.io.tmpdir") "/" target-name)) 115 | (assoc :name target-name)) 116 | _ (println "Target config" target) 117 | {:keys [t]} (timed (w/migrate source target))] 118 | (print-results {:t t 119 | :date (Date.) 120 | :options options} output))) 121 | (let [_ (println "No source given. Using random databases...") 122 | source-name (str "wanderung_s_" (random-str 8)) 123 | source {:wanderung/type :datahike 124 | :store (merge {:backend backend} 125 | (when (= :mem backend) 126 | {:id source-name}) 127 | (when (= :file backend) 128 | {:path (str (System/getProperty "java.io.tmpdir") "/" source-name)})) 129 | :keep-history? true 130 | :name source-name 131 | :schema-flexibility :write 132 | :attribute-refs? (some? (:refs options)) 133 | :index (keyword "datahike.index" (name index))} 134 | _ (println "Source config" source) 135 | _ (init-source source tx-count upsert-count entities-count) 136 | target-name (str "wanderung_t_" (random-str 8)) 137 | target (-> source 138 | #_(assoc-in [:store :path] (str (System/getProperty "java.io.tmpdir") "/" target-name)) 139 | (assoc-in [:store :id] target-name) 140 | (assoc :name target-name)) 141 | _ (println "Target config" target) 142 | results (vec (repeatedly sample (fn [] 143 | (d/delete-database target) 144 | (d/create-database target) 145 | (timed (w/migrate source target)))))] 146 | (println "Cleaning up random databases...") 147 | (d/delete-database source) 148 | (d/delete-database target) 149 | (println "Done") 150 | (print-results {:t (avg-variance results) 151 | :date (Date.) 152 | :options options} output)))))))) 153 | 154 | (comment 155 | 156 | 157 | (-main "-p" "5" "-n" "1000" "-u" "1" "-e" "1" "-i" "persistent-set" "-b" "mem" "-r") 158 | 159 | (-main "-h") 160 | 161 | 162 | 163 | ) 164 | -------------------------------------------------------------------------------- /test/wanderung/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns wanderung.core-test 2 | (:require [clojure.test :refer :all] 3 | [wanderung.core :as wanderung] 4 | [datahike.api :as dh] 5 | [datomic.client.api :as dt] 6 | [clojure.edn :as edn] 7 | [taoensso.timbre :as timbre] 8 | [clojure.java.io :as io]) 9 | (:import [java.io File])) 10 | 11 | (timbre/set-level! :warn) 12 | 13 | (def testdata (-> "testdatoms.edn" 14 | io/resource 15 | slurp 16 | edn/read-string)) 17 | 18 | (defn datomic-cfg [db-name] 19 | {:wanderung/type :datomic 20 | :server-type :dev-local 21 | :storage-dir :mem 22 | :name db-name 23 | :system "CI"}) 24 | 25 | (defn setup-datomic-conn 26 | "Given a database name creates a new datomic in-memory database and returns a connection." 27 | [db-name] 28 | (let [client-cfg (datomic-cfg db-name) 29 | db-cfg {:db-name db-name} 30 | dt-client (dt/client client-cfg) 31 | _ (dt/delete-database dt-client db-cfg) 32 | _ (dt/create-database dt-client db-cfg)] 33 | (dt/connect dt-client db-cfg))) 34 | 35 | (defn datahike-cfg [db-name] 36 | {:wanderung/type :datahike 37 | :store {:backend :mem 38 | :id db-name}}) 39 | 40 | (defn setup-datahike-conn 41 | "Given a database name creates a new datahike in-memory database and returns a connection." 42 | [db-name] 43 | (let [cfg (datahike-cfg db-name)] 44 | (dh/delete-database cfg) 45 | (dh/create-database cfg) 46 | (dh/connect cfg))) 47 | 48 | (defn setup-data [tx-fn conn] 49 | (tx-fn conn {:tx-data [{:db/ident :person/name 50 | :db/valueType :db.type/string 51 | :db/unique :db.unique/identity 52 | :db/cardinality :db.cardinality/one} 53 | {:db/ident :person/age 54 | :db/valueType :db.type/long 55 | :db/cardinality :db.cardinality/one} 56 | {:db/ident :person/siblings 57 | :db/valueType :db.type/ref 58 | :db/cardinality :db.cardinality/many}]}) 59 | (tx-fn conn {:tx-data [{:db/id -1 60 | :person/name "Alice" 61 | :person/age 25} 62 | {:db/id -2 63 | :person/name "Bob" 64 | :person/age 35} 65 | {:person/name "Charlie" 66 | :person/age 45 67 | :person/siblings [-1 -2]}]})) 68 | 69 | (deftest test-nippy-migration 70 | (let [a {:wanderung/type :nippy 71 | :filename (str (File/createTempFile "a_file" ".nippy"))} 72 | b {:wanderung/type :nippy 73 | :filename (str (File/createTempFile "b_file" ".nippy"))}] 74 | (wanderung/datoms-to-storage a testdata) 75 | (wanderung/migrate a b) 76 | (is (= testdata (wanderung/datoms-from-storage b))))) 77 | 78 | (deftest test-datomic->datahike-basic 79 | (let [db-name "dt->dh-test-basic" 80 | dt-conn (setup-datomic-conn db-name)] 81 | (setup-datahike-conn db-name) 82 | (setup-data dt/transact dt-conn) 83 | (wanderung/migrate (datomic-cfg db-name) (datahike-cfg db-name)) 84 | (testing "test basic data and query" 85 | (letfn [(coerce-result [result] 86 | (->> result 87 | (map #(update (first %) :person/siblings set)) 88 | set))] 89 | (let [dh-conn (dh/connect (datahike-cfg db-name)) 90 | query '[:find (pull ?e [:person/name :person/age {:person/siblings [:person/name]}]) 91 | :where [?e :person/name _]] 92 | dt-result (->> (dt/db dt-conn) 93 | (dt/q query) 94 | coerce-result) 95 | dh-result (->> (dh/db dh-conn) 96 | (dh/q query) 97 | coerce-result)] 98 | (is (= dt-result 99 | dh-result))))))) 100 | 101 | (deftest test-datomic->datahike-history 102 | (let [db-name "dt->dh-test-history" 103 | dt-conn (setup-datomic-conn db-name)] 104 | (setup-datahike-conn db-name) 105 | (setup-data dt/transact dt-conn) 106 | (dt/transact dt-conn {:tx-data [[:db/retractEntity [:person/name "Alice"]]]}) 107 | (wanderung/migrate (datomic-cfg db-name) (datahike-cfg db-name)) 108 | (let [dh-conn (dh/connect (datahike-cfg db-name)) 109 | dh-db (dh/db dh-conn) 110 | dt-db (dt/db dt-conn)] 111 | (testing "current snapshot without retracted data" 112 | (letfn [(coerce-result [result] 113 | (->> result 114 | (map #(update (first %) :person/siblings set)) 115 | set))] 116 | (let [query '[:find (pull ?e [:person/name :person/age {:person/siblings [:person/name]}]) 117 | :where [?e :person/name _]] 118 | dt-result (->> dt-db 119 | (dt/q query) 120 | coerce-result) 121 | dh-result (->> dh-db 122 | (dh/q query) 123 | coerce-result)] 124 | (is (= dt-result 125 | dh-result))))) 126 | (testing "history snapshot with retraction time" 127 | (let [query '[:find ?n ?d ?op 128 | :where 129 | [?e :person/name ?n ?t ?op] 130 | [?t :db/txInstant ?d]] 131 | dt-result (->> dt-db 132 | dt/history 133 | (dt/q query) 134 | set) 135 | dh-result (->> dh-db 136 | dh/history 137 | (dh/q query) 138 | set)] 139 | (is (= dt-result 140 | dh-result)))) 141 | (testing "schema" 142 | (let [query '[:find (pull ?e [*]) 143 | :in $ [?attrs ...] 144 | :where 145 | [?e :db/ident ?attrs]] 146 | attributes [:person/name :person/age :person/siblings] 147 | dt-result (->> (dt/q query 148 | dt-db 149 | attributes) 150 | (map first) 151 | (map (fn [{:keys [db/unique] :as attr}] 152 | (cond-> attr 153 | true (dissoc :db/id) 154 | true (update :db/valueType :db/ident) 155 | true (update :db/cardinality :db/ident) 156 | (some? unique) (update :db/unique :db/ident)))) 157 | set) 158 | dh-result (->> (dh/q query 159 | dh-db 160 | attributes) 161 | (map (comp #(dissoc % :db/id) first)) 162 | set)] 163 | (is (= dt-result 164 | dh-result))))))) 165 | 166 | (deftest test-datahike->datomic-basic 167 | (let [db-name "dh->dt-test-basic" 168 | dh-conn (setup-datahike-conn db-name)] 169 | (setup-datomic-conn db-name) 170 | (setup-data dh/transact dh-conn) 171 | (wanderung/migrate (datahike-cfg db-name) (datomic-cfg db-name)) 172 | (testing "test basic data and query" 173 | (letfn [(coerce-result [result] 174 | (->> result 175 | (map #(update (first %) :person/siblings set)) 176 | set))] 177 | (let [dt-conn (dt/connect (dt/client (datomic-cfg db-name)) {:db-name db-name}) 178 | query '[:find (pull ?e [:person/name :person/age {:person/siblings [:person/name]}]) 179 | :where [?e :person/name _]] 180 | dh-result (->> (dh/db dh-conn) 181 | (dh/q query) 182 | coerce-result) 183 | dt-result (->> (dt/db dt-conn) 184 | (dt/q query) 185 | coerce-result)] 186 | (is (= dh-result 187 | dt-result))))))) 188 | 189 | (deftest test-datahike->datomic-history 190 | (let [db-name "dh->dt-test-history" 191 | dh-conn (setup-datahike-conn db-name)] 192 | (setup-datomic-conn db-name) 193 | (setup-data dh/transact dh-conn) 194 | (dh/transact dh-conn {:tx-data [[:db/retractEntity [:person/name "Alice"]]]}) 195 | (wanderung/migrate (datahike-cfg db-name) (datomic-cfg db-name)) 196 | (let [dt-conn (dt/connect (dt/client (datomic-cfg db-name)) {:db-name db-name}) 197 | dt-db (dt/db dt-conn) 198 | dh-db (dh/db dh-conn)] 199 | (testing "current snapshot without retracted data" 200 | (letfn [(coerce-result [result] 201 | (->> result 202 | (map #(update (first %) :person/siblings set)) 203 | set))] 204 | (let [query '[:find (pull ?e [:person/name :person/age {:person/siblings [:person/name]}]) 205 | :where [?e :person/name _]] 206 | dh-result (->> dh-db 207 | (dh/q query) 208 | coerce-result) 209 | dt-result (->> dt-db 210 | (dt/q query) 211 | coerce-result)] 212 | (is (= dh-result 213 | dt-result))))) 214 | (testing "history snapshot with retraction time" 215 | (let [query '[:find ?n ?d ?op 216 | :where 217 | [?e :person/name ?n ?t ?op] 218 | [?t :db/txInstant ?d]] 219 | dh-result (->> dh-db 220 | dh/history 221 | (dh/q query) 222 | set) 223 | dt-result (->> dt-db 224 | dt/history 225 | (dt/q query) 226 | set)] 227 | (is (= dh-result 228 | dt-result)))) 229 | (testing "schema" 230 | (let [query '[:find (pull ?e [*]) 231 | :in $ [?attrs ...] 232 | :where 233 | [?e :db/ident ?attrs]] 234 | attributes [:person/name :person/age :person/siblings] 235 | dh-result (->> (dh/q query 236 | dh-db 237 | attributes) 238 | (map (comp #(dissoc % :db/id) first)) 239 | set) 240 | dt-result (->> (dt/q query 241 | dt-db 242 | attributes) 243 | (map first) 244 | (map (fn [{:keys [db/unique] :as attr}] 245 | (cond-> attr 246 | true (dissoc :db/id) 247 | true (update :db/valueType :db/ident) 248 | true (update :db/cardinality :db/ident) 249 | (some? unique) (update :db/unique :db/ident)))) 250 | set)] 251 | (is (= dh-result 252 | dt-result))))))) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Eclipse Public License - v 2.0 2 | 3 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE 4 | PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION 5 | OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 6 | 7 | 1. DEFINITIONS 8 | 9 | "Contribution" means: 10 | 11 | a) in the case of the initial Contributor, the initial content 12 | Distributed under this Agreement, and 13 | 14 | b) in the case of each subsequent Contributor: 15 | i) changes to the Program, and 16 | ii) additions to the Program; 17 | where such changes and/or additions to the Program originate from 18 | and are Distributed by that particular Contributor. A Contribution 19 | "originates" from a Contributor if it was added to the Program by 20 | such Contributor itself or anyone acting on such Contributor's behalf. 21 | Contributions do not include changes or additions to the Program that 22 | are not Modified Works. 23 | 24 | "Contributor" means any person or entity that Distributes the Program. 25 | 26 | "Licensed Patents" mean patent claims licensable by a Contributor which 27 | are necessarily infringed by the use or sale of its Contribution alone 28 | or when combined with the Program. 29 | 30 | "Program" means the Contributions Distributed in accordance with this 31 | Agreement. 32 | 33 | "Recipient" means anyone who receives the Program under this Agreement 34 | or any Secondary License (as applicable), including Contributors. 35 | 36 | "Derivative Works" shall mean any work, whether in Source Code or other 37 | form, that is based on (or derived from) the Program and for which the 38 | editorial revisions, annotations, elaborations, or other modifications 39 | represent, as a whole, an original work of authorship. 40 | 41 | "Modified Works" shall mean any work in Source Code or other form that 42 | results from an addition to, deletion from, or modification of the 43 | contents of the Program, including, for purposes of clarity any new file 44 | in Source Code form that contains any contents of the Program. Modified 45 | Works shall not include works that contain only declarations, 46 | interfaces, types, classes, structures, or files of the Program solely 47 | in each case in order to link to, bind by name, or subclass the Program 48 | or Modified Works thereof. 49 | 50 | "Distribute" means the acts of a) distributing or b) making available 51 | in any manner that enables the transfer of a copy. 52 | 53 | "Source Code" means the form of a Program preferred for making 54 | modifications, including but not limited to software source code, 55 | documentation source, and configuration files. 56 | 57 | "Secondary License" means either the GNU General Public License, 58 | Version 2.0, or any later versions of that license, including any 59 | exceptions or additional permissions as identified by the initial 60 | Contributor. 61 | 62 | 2. GRANT OF RIGHTS 63 | 64 | a) Subject to the terms of this Agreement, each Contributor hereby 65 | grants Recipient a non-exclusive, worldwide, royalty-free copyright 66 | license to reproduce, prepare Derivative Works of, publicly display, 67 | publicly perform, Distribute and sublicense the Contribution of such 68 | Contributor, if any, and such Derivative Works. 69 | 70 | b) Subject to the terms of this Agreement, each Contributor hereby 71 | grants Recipient a non-exclusive, worldwide, royalty-free patent 72 | license under Licensed Patents to make, use, sell, offer to sell, 73 | import and otherwise transfer the Contribution of such Contributor, 74 | if any, in Source Code or other form. This patent license shall 75 | apply to the combination of the Contribution and the Program if, at 76 | the time the Contribution is added by the Contributor, such addition 77 | of the Contribution causes such combination to be covered by the 78 | Licensed Patents. The patent license shall not apply to any other 79 | combinations which include the Contribution. No hardware per se is 80 | licensed hereunder. 81 | 82 | c) Recipient understands that although each Contributor grants the 83 | licenses to its Contributions set forth herein, no assurances are 84 | provided by any Contributor that the Program does not infringe the 85 | patent or other intellectual property rights of any other entity. 86 | Each Contributor disclaims any liability to Recipient for claims 87 | brought by any other entity based on infringement of intellectual 88 | property rights or otherwise. As a condition to exercising the 89 | rights and licenses granted hereunder, each Recipient hereby 90 | assumes sole responsibility to secure any other intellectual 91 | property rights needed, if any. For example, if a third party 92 | patent license is required to allow Recipient to Distribute the 93 | Program, it is Recipient's responsibility to acquire that license 94 | before distributing the Program. 95 | 96 | d) Each Contributor represents that to its knowledge it has 97 | sufficient copyright rights in its Contribution, if any, to grant 98 | the copyright license set forth in this Agreement. 99 | 100 | e) Notwithstanding the terms of any Secondary License, no 101 | Contributor makes additional grants to any Recipient (other than 102 | those set forth in this Agreement) as a result of such Recipient's 103 | receipt of the Program under the terms of a Secondary License 104 | (if permitted under the terms of Section 3). 105 | 106 | 3. REQUIREMENTS 107 | 108 | 3.1 If a Contributor Distributes the Program in any form, then: 109 | 110 | a) the Program must also be made available as Source Code, in 111 | accordance with section 3.2, and the Contributor must accompany 112 | the Program with a statement that the Source Code for the Program 113 | is available under this Agreement, and informs Recipients how to 114 | obtain it in a reasonable manner on or through a medium customarily 115 | used for software exchange; and 116 | 117 | b) the Contributor may Distribute the Program under a license 118 | different than this Agreement, provided that such license: 119 | i) effectively disclaims on behalf of all other Contributors all 120 | warranties and conditions, express and implied, including 121 | warranties or conditions of title and non-infringement, and 122 | implied warranties or conditions of merchantability and fitness 123 | for a particular purpose; 124 | 125 | ii) effectively excludes on behalf of all other Contributors all 126 | liability for damages, including direct, indirect, special, 127 | incidental and consequential damages, such as lost profits; 128 | 129 | iii) does not attempt to limit or alter the recipients' rights 130 | in the Source Code under section 3.2; and 131 | 132 | iv) requires any subsequent distribution of the Program by any 133 | party to be under a license that satisfies the requirements 134 | of this section 3. 135 | 136 | 3.2 When the Program is Distributed as Source Code: 137 | 138 | a) it must be made available under this Agreement, or if the 139 | Program (i) is combined with other material in a separate file or 140 | files made available under a Secondary License, and (ii) the initial 141 | Contributor attached to the Source Code the notice described in 142 | Exhibit A of this Agreement, then the Program may be made available 143 | under the terms of such Secondary Licenses, and 144 | 145 | b) a copy of this Agreement must be included with each copy of 146 | the Program. 147 | 148 | 3.3 Contributors may not remove or alter any copyright, patent, 149 | trademark, attribution notices, disclaimers of warranty, or limitations 150 | of liability ("notices") contained within the Program from any copy of 151 | the Program which they Distribute, provided that Contributors may add 152 | their own appropriate notices. 153 | 154 | 4. COMMERCIAL DISTRIBUTION 155 | 156 | Commercial distributors of software may accept certain responsibilities 157 | with respect to end users, business partners and the like. While this 158 | license is intended to facilitate the commercial use of the Program, 159 | the Contributor who includes the Program in a commercial product 160 | offering should do so in a manner which does not create potential 161 | liability for other Contributors. Therefore, if a Contributor includes 162 | the Program in a commercial product offering, such Contributor 163 | ("Commercial Contributor") hereby agrees to defend and indemnify every 164 | other Contributor ("Indemnified Contributor") against any losses, 165 | damages and costs (collectively "Losses") arising from claims, lawsuits 166 | and other legal actions brought by a third party against the Indemnified 167 | Contributor to the extent caused by the acts or omissions of such 168 | Commercial Contributor in connection with its distribution of the Program 169 | in a commercial product offering. The obligations in this section do not 170 | apply to any claims or Losses relating to any actual or alleged 171 | intellectual property infringement. In order to qualify, an Indemnified 172 | Contributor must: a) promptly notify the Commercial Contributor in 173 | writing of such claim, and b) allow the Commercial Contributor to control, 174 | and cooperate with the Commercial Contributor in, the defense and any 175 | related settlement negotiations. The Indemnified Contributor may 176 | participate in any such claim at its own expense. 177 | 178 | For example, a Contributor might include the Program in a commercial 179 | product offering, Product X. That Contributor is then a Commercial 180 | Contributor. If that Commercial Contributor then makes performance 181 | claims, or offers warranties related to Product X, those performance 182 | claims and warranties are such Commercial Contributor's responsibility 183 | alone. Under this section, the Commercial Contributor would have to 184 | defend claims against the other Contributors related to those performance 185 | claims and warranties, and if a court requires any other Contributor to 186 | pay any damages as a result, the Commercial Contributor must pay 187 | those damages. 188 | 189 | 5. NO WARRANTY 190 | 191 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT 192 | PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" 193 | BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 194 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF 195 | TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR 196 | PURPOSE. Each Recipient is solely responsible for determining the 197 | appropriateness of using and distributing the Program and assumes all 198 | risks associated with its exercise of rights under this Agreement, 199 | including but not limited to the risks and costs of program errors, 200 | compliance with applicable laws, damage to or loss of data, programs 201 | or equipment, and unavailability or interruption of operations. 202 | 203 | 6. DISCLAIMER OF LIABILITY 204 | 205 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT 206 | PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS 207 | SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 208 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST 209 | PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 210 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 211 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 212 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE 213 | POSSIBILITY OF SUCH DAMAGES. 214 | 215 | 7. GENERAL 216 | 217 | If any provision of this Agreement is invalid or unenforceable under 218 | applicable law, it shall not affect the validity or enforceability of 219 | the remainder of the terms of this Agreement, and without further 220 | action by the parties hereto, such provision shall be reformed to the 221 | minimum extent necessary to make such provision valid and enforceable. 222 | 223 | If Recipient institutes patent litigation against any entity 224 | (including a cross-claim or counterclaim in a lawsuit) alleging that the 225 | Program itself (excluding combinations of the Program with other software 226 | or hardware) infringes such Recipient's patent(s), then such Recipient's 227 | rights granted under Section 2(b) shall terminate as of the date such 228 | litigation is filed. 229 | 230 | All Recipient's rights under this Agreement shall terminate if it 231 | fails to comply with any of the material terms or conditions of this 232 | Agreement and does not cure such failure in a reasonable period of 233 | time after becoming aware of such noncompliance. If all Recipient's 234 | rights under this Agreement terminate, Recipient agrees to cease use 235 | and distribution of the Program as soon as reasonably practicable. 236 | However, Recipient's obligations under this Agreement and any licenses 237 | granted by Recipient relating to the Program shall continue and survive. 238 | 239 | Everyone is permitted to copy and distribute copies of this Agreement, 240 | but in order to avoid inconsistency the Agreement is copyrighted and 241 | may only be modified in the following manner. The Agreement Steward 242 | reserves the right to publish new versions (including revisions) of 243 | this Agreement from time to time. No one other than the Agreement 244 | Steward has the right to modify this Agreement. The Eclipse Foundation 245 | is the initial Agreement Steward. The Eclipse Foundation may assign the 246 | responsibility to serve as the Agreement Steward to a suitable separate 247 | entity. Each new version of the Agreement will be given a distinguishing 248 | version number. The Program (including Contributions) may always be 249 | Distributed subject to the version of the Agreement under which it was 250 | received. In addition, after a new version of the Agreement is published, 251 | Contributor may elect to Distribute the Program (including its 252 | Contributions) under the new version. 253 | 254 | Except as expressly stated in Sections 2(a) and 2(b) above, Recipient 255 | receives no rights or licenses to the intellectual property of any 256 | Contributor under this Agreement, whether expressly, by implication, 257 | estoppel or otherwise. All rights in the Program not expressly granted 258 | under this Agreement are reserved. Nothing in this Agreement is intended 259 | to be enforceable by any entity that is not a Contributor or Recipient. 260 | No third-party beneficiary rights are created under this Agreement. 261 | 262 | Exhibit A - Form of Secondary Licenses Notice 263 | 264 | "This Source Code may also be made available under the following 265 | Secondary Licenses when the conditions for such availability set forth 266 | in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), 267 | version(s), and exceptions or additional permissions here}." 268 | 269 | Simply including a copy of this Agreement, including this Exhibit A 270 | is not sufficient to license the Source Code under Secondary Licenses. 271 | 272 | If it is not possible or desirable to put the notice in a particular 273 | file, then You may include the notice in a location (such as a LICENSE 274 | file in a relevant directory) where a recipient would be likely to 275 | look for such a notice. 276 | 277 | You may add additional accurate notices of copyright ownership. 278 | --------------------------------------------------------------------------------