├── bin └── kaocha ├── .gitignore ├── Makefile ├── tests.edn ├── project.clj ├── src └── datoms_differ │ ├── export.cljc │ ├── impl │ └── core_helpers.cljc │ ├── datom.cljc │ ├── core.cljc │ ├── reporter.clj │ └── api.cljc ├── test └── datoms_differ │ ├── export_test.cljc │ ├── impl │ └── core_helpers_test.cljc │ ├── reporter_test.clj │ ├── core_test.cljc │ └── api_test.cljc ├── README.md └── dev └── perf.clj /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | lein kaocha "$@" 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | /pom.xml 3 | /pom.xml.asc 4 | /.nrepl-port 5 | /.clj-kondo 6 | /.lsp -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | deploy: 2 | lein deploy clojars 3 | 4 | test: 5 | bin/kaocha 6 | 7 | .PHONY: test deploy 8 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:plugins [:noyoda.plugin/swap-actual-and-expected] 3 | :tests [{:id :unit 4 | :source-paths ["src"] 5 | :focus-meta [:focus]}]} 6 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject datoms-differ "2025-11-14" 2 | :description "Find the diff between two txes in datoms." 3 | :url "http://github.com/magnars/datoms-differ" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[dev.weavejester/medley "1.9.0"] 7 | [clansi "1.0.0"] 8 | [persistent-sorted-set "0.3.0"]] 9 | :profiles {:dev {:dependencies [[org.clojure/clojure "1.10.1"] 10 | [org.clojure/tools.cli "0.4.2"] ;; for kaocha to recognize command line options 11 | [lambdaisland/kaocha "0.0-590"] 12 | [kaocha-noyoda "2019-06-03"] 13 | [criterium "0.4.5"] 14 | [com.taoensso/tufte "2.1.0"]]}} 15 | :aliases {"kaocha" ["run" "-m" "kaocha.runner"]}) 16 | -------------------------------------------------------------------------------- /src/datoms_differ/export.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.export 2 | (:require [datoms-differ.core :refer [get-datoms]] 3 | [datoms-differ.impl.core-helpers :as ch] 4 | [medley.core :refer [filter-keys map-vals]])) 5 | 6 | (def tx0 (inc (:to datoms-differ.core/default-db-id-partition))) 7 | 8 | (defn export [schema datoms & {:keys [partition-key start-tx] 9 | :or {partition-key :datoms-differ.core/db-id-partition 10 | start-tx tx0}}] 11 | (str "#datascript/DB " 12 | (pr-str {:schema (dissoc schema partition-key) 13 | :datoms (into [] (map #(conj % start-tx)) datoms)}))) 14 | 15 | 16 | (defn prep-for-datascript 17 | "Filter away any keys from the schema contents that does not start with the :db/ namespace" 18 | [schema] 19 | (map-vals #(filter-keys (fn [k] (= (namespace k) "db")) %) schema)) 20 | 21 | (defn ^:export export-db [db] 22 | (export (prep-for-datascript (:schema db)) (get-datoms db))) 23 | 24 | (defn prune-diffs [schema tx-data] 25 | (let [{:keys [many?]} (ch/find-attrs schema) 26 | overwriting-additions (set (keep (fn [[op eid attr]] 27 | (when (and (= :db/add op) 28 | (not (many? attr))) 29 | [eid attr])) 30 | tx-data))] 31 | (remove (fn [[op eid attr]] 32 | (and (= :db/retract op) 33 | (overwriting-additions [eid attr]))) 34 | tx-data))) 35 | 36 | -------------------------------------------------------------------------------- /test/datoms_differ/export_test.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.export-test 2 | (:require [clojure.test :refer [deftest is]] 3 | [datoms-differ.core-test :refer [schema]] 4 | [datoms-differ.export :as sut])) 5 | 6 | (deftest exports-schema-and-datoms 7 | (is (= (sut/export schema [[2024 :route/number "100"] 8 | [2024 :route/services 2025] 9 | [2025 :service/id :s567] 10 | [2025 :service/allocated-vessel 2026] 11 | [2026 :vessel/imo "123"]]) 12 | (str "#datascript/DB {" 13 | ":schema " 14 | "{:route/tags #:db{:cardinality :db.cardinality/many}," 15 | " :service/label {}," 16 | " :route/name {}," 17 | " :trip/id #:db{:unique :db.unique/identity}," 18 | " :vessel/imo #:db{:unique :db.unique/identity}," 19 | " :service/trips #:db{:valueType :db.type/ref, :cardinality :db.cardinality/many, :isComponent true}," 20 | " :vessel/name {}," 21 | " :route/services #:db{:valueType :db.type/ref, :cardinality :db.cardinality/many}," 22 | " :service/allocated-vessel #:db{:valueType :db.type/ref, :cardinality :db.cardinality/one}," 23 | " :route/number #:db{:unique :db.unique/identity}," 24 | " :service/id #:db{:unique :db.unique/identity}}, " 25 | ":datoms " 26 | "[[2024 :route/number \"100\" 536870912]" 27 | " [2024 :route/services 2025 536870912]" 28 | " [2025 :service/id :s567 536870912]" 29 | " [2025 :service/allocated-vessel 2026 536870912]" 30 | " [2026 :vessel/imo \"123\" 536870912]]" 31 | "}")))) 32 | 33 | (deftest prunes-diffs 34 | (is (= (sut/prune-diffs schema 35 | [[:db/retract 2025 :service/allocated-vessel 2026] 36 | [:db/add 2025 :service/allocated-vessel 2027]]) 37 | [[:db/add 2025 :service/allocated-vessel 2027]])) 38 | 39 | (is (= (sut/prune-diffs schema 40 | [[:db/retract 2025 :route/services 2026] 41 | [:db/add 2025 :route/services 2027]]) 42 | [[:db/retract 2025 :route/services 2026] 43 | [:db/add 2025 :route/services 2027]]))) 44 | 45 | (deftest preps-schema-for-datascript 46 | (is (= (sut/prep-for-datascript {:a/id {:db/unique :db.unique/identity} 47 | :a/name {:spec string?} 48 | :a/owner {} 49 | :a/address {:db/valueType :db.type/ref 50 | :spec coll?}}) 51 | {:a/id {:db/unique :db.unique/identity} 52 | :a/name {} 53 | :a/owner {} 54 | :a/address {:db/valueType :db.type/ref}}))) 55 | -------------------------------------------------------------------------------- /src/datoms_differ/impl/core_helpers.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.impl.core-helpers 2 | (:import (clojure.lang RT))) 3 | 4 | (defn- find-attr [schema target-k target-v] 5 | (set (keep (fn [[k v]] 6 | (when (= target-v (target-k v)) 7 | k)) 8 | schema))) 9 | 10 | (defn- validate-attrs [{:keys [many? tuple?]}] 11 | (doseq [[k tupleAttrs] tuple?] 12 | (doseq [tupleAttr tupleAttrs] 13 | (when (many? tupleAttr) 14 | (throw (ex-info "Tuple attribute can't reference cardinality many attribute" 15 | {:attr k 16 | :conflict #{tupleAttr}}))) 17 | (when (tuple? tupleAttr) 18 | (throw (ex-info "Tuple attribute can't reference another tuple attribute" 19 | {:attr k 20 | :conflict #{tupleAttr}})))))) 21 | 22 | (defn find-attrs 23 | "Find relevant attrs grouped by attribute type" 24 | [schema] 25 | (let [attrs {:identity? (find-attr schema :db/unique :db.unique/identity) 26 | :ref? (find-attr schema :db/valueType :db.type/ref) 27 | :many? (find-attr schema :db/cardinality :db.cardinality/many) 28 | :component? (find-attr schema :db/isComponent true) 29 | :tuple? (into {} (keep (fn [[k v]] 30 | (when-let [tupleAttrs (:db/tupleAttrs v)] 31 | [k tupleAttrs])) 32 | schema))}] 33 | (validate-attrs attrs) 34 | attrs)) 35 | 36 | (defn add-tuple-attributes [attrs entity] 37 | (reduce-kv (fn [e k tupleAttrs] 38 | (if (some entity tupleAttrs) 39 | (assoc e k (mapv #(get e %) tupleAttrs)) 40 | e)) 41 | entity 42 | (:tuple? attrs))) 43 | 44 | (defn get-entity-ref [attrs entity] 45 | (let [refs (select-keys entity (conj (:identity? attrs) :db/id))] 46 | (case (bounded-count 2 refs) 47 | 0 (throw (ex-info "Entity without identity attribute" 48 | {:entity entity 49 | :attrs (:identity attrs)})) 50 | 1 (first refs) 51 | 2 (throw (ex-info (str "Entity with multiple identity attributes: " refs) 52 | {:entity entity 53 | :attrs (:identity? attrs)}))))) 54 | 55 | (defn- select-first-entry-of [map keyseq] 56 | (loop [ret nil keys (seq keyseq)] 57 | (when keys 58 | (if-let [entry (. RT (find map (first keys)))] 59 | entry 60 | (recur ret (next keys)))))) 61 | 62 | (defn get-entity-ref-unsafe [attrs entity-map] 63 | (select-first-entry-of entity-map (:identity? attrs))) 64 | 65 | (defn reverse-ref? [k] 66 | (.startsWith (name k) "_")) 67 | 68 | (defn reverse-ref-attr [k] 69 | (if (reverse-ref? k) 70 | (keyword (namespace k) (subs (name k) 1)) 71 | (keyword (namespace k) (str "_" (name k))))) 72 | 73 | (defn find-all-entities [{:keys [ref? many? component?] :as attrs} entity-maps] 74 | (persistent! 75 | (reduce 76 | (fn [res entity] 77 | (reduce-kv 78 | (fn [res k v] 79 | (reduce conj! res 80 | (cond 81 | (ref? k) (find-all-entities attrs (if (many? k) v [v])) 82 | (reverse-ref? k) (let [reverse-k (reverse-ref-attr k)] 83 | (find-all-entities attrs (if (component? reverse-k) [v] v))) 84 | :else []))) 85 | res 86 | entity)) 87 | (transient (into [] entity-maps)) 88 | entity-maps))) 89 | -------------------------------------------------------------------------------- /test/datoms_differ/impl/core_helpers_test.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.impl.core-helpers-test 2 | (:require [datoms-differ.impl.core-helpers :as sut] 3 | [clojure.test :refer [deftest is testing]])) 4 | 5 | (def schema 6 | {:route/name {} 7 | :route/number {:db/unique :db.unique/identity} 8 | :route/services {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many} 9 | :route/tags {:db/cardinality :db.cardinality/many} 10 | :route/holidays {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many} 11 | :service/trips {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/isComponent true} 12 | :trip/id {:db/unique :db.unique/identity} 13 | :service/id {:db/unique :db.unique/identity} 14 | :service/label {} 15 | :service/allocated-vessel {:db/valueType :db.type/ref :db/cardinality :db.cardinality/one} 16 | :vessel/imo {:db/unique :db.unique/identity} 17 | :vessel/name {} 18 | :holiday/bus-key {:db/unique :db.unique/identity 19 | :db/tupleAttrs [:holiday/pattern :holiday/weekday]} 20 | :holiday/pattern {} 21 | :holiday/weekday {}}) 22 | 23 | (deftest finds-attrs 24 | (is (= (sut/find-attrs schema) 25 | {:identity? #{:route/number :service/id :vessel/imo :trip/id :holiday/bus-key} 26 | :ref? #{:route/services :service/allocated-vessel :service/trips :route/holidays} 27 | :many? #{:route/services :service/trips :route/tags :route/holidays} 28 | :component? #{:service/trips} 29 | :tuple? {:holiday/bus-key [:holiday/pattern :holiday/weekday]}})) 30 | 31 | (testing "tuple attribute referencing many attribute throws" 32 | (try 33 | (sut/find-attrs {:a-many {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many} 34 | :b {} 35 | :tup {:db/tupleAttrs [:a-many :b]}}) 36 | (is (= :should-throw :didnt)) 37 | (catch Exception e 38 | (is (= "Tuple attribute can't reference cardinality many attribute" (.getMessage e))) 39 | (is (= {:attr :tup 40 | :conflict #{:a-many}} 41 | (ex-data e)))))) 42 | 43 | (testing "tuple attribute referencing another tuple attribute throws" 44 | (try 45 | (sut/find-attrs {:a {} 46 | :b {} 47 | :tup {:db/tupleAttrs [:a :b]} 48 | :c {} 49 | :tup2 {:db/tupleAttrs [:c :tup]}}) 50 | (is (= :should-throw :didnt)) 51 | (catch Exception e 52 | (is (= "Tuple attribute can't reference another tuple attribute" (.getMessage e))) 53 | (is (= {:attr :tup2 54 | :conflict #{:tup}} 55 | (ex-data e))))))) 56 | 57 | (def attrs (sut/find-attrs schema)) 58 | 59 | (deftest add-tuple-attributes 60 | (is (= (sut/add-tuple-attributes attrs {:holiday/pattern :christmas-day :holiday/weekday :sunday}) 61 | {:holiday/bus-key [:christmas-day :sunday] 62 | :holiday/pattern :christmas-day 63 | :holiday/weekday :sunday})) 64 | (is (= (sut/add-tuple-attributes attrs {:holiday/weekday :sunday}) 65 | {:holiday/bus-key [nil :sunday] 66 | :holiday/weekday :sunday})) 67 | (is (= (sut/add-tuple-attributes attrs {:foo :bar}) 68 | {:foo :bar}))) 69 | 70 | (deftest gets-entity-refs 71 | (is (= (sut/get-entity-ref attrs {:route/number "100" :route/name "Stavanger-Tau"}) 72 | [:route/number "100"])) 73 | 74 | (is (= (->> {:holiday/pattern :christmas-eve :holiday/weekday :sunday} 75 | (sut/add-tuple-attributes attrs) 76 | (sut/get-entity-ref attrs)) 77 | [:holiday/bus-key [:christmas-eve :sunday]])) 78 | 79 | (is (thrown? Exception ;; multiple identity attributes 80 | (sut/get-entity-ref attrs {:route/number "100" :service/id 200 :route/name "Stavanger-Tau"}))) 81 | 82 | (is (thrown? Exception ;; no identity attributes 83 | (sut/get-entity-ref attrs {:route/name "Stavanger-Tau"}))) 84 | 85 | (is (= (sut/get-entity-ref attrs {:db/id 123456 :route/name "100"}) 86 | [:db/id 123456]))) 87 | 88 | (deftest finds-all-entities 89 | (is (= (sut/find-all-entities attrs [{:route/number "100" 90 | :route/services [{:service/id :s567 91 | :service/allocated-vessel {:vessel/imo "123"}}]}]) 92 | [{:route/number "100" 93 | :route/services [{:service/id :s567 94 | :service/allocated-vessel {:vessel/imo "123"}}]} 95 | {:service/id :s567 96 | :service/allocated-vessel {:vessel/imo "123"}} 97 | {:vessel/imo "123"}])) 98 | 99 | (is (= (sut/find-all-entities attrs [{:vessel/imo "123" 100 | :service/_allocated-vessel [{:service/id :s567}]}]) 101 | [{:vessel/imo "123" 102 | :service/_allocated-vessel [{:service/id :s567}]} 103 | {:service/id :s567}]))) 104 | 105 | 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # datoms-differ 2 | 3 | Find the diff (as datoms) between two txes (of entity maps). 4 | 5 | *What the what?* 6 | 7 | Yeah, good question. Ahem. Basically, it's a way of keeping track of changes in 8 | data via a datomic/datascript-like API and data structure. 9 | 10 | *What's it for then?* 11 | 12 | One use case is keeping clients in sync with a backend. The 13 | client is initialized with a datascript db when connecting, and is fed updates 14 | in the form of tx-data when the backend changes. 15 | 16 | Or you might just be looking to diff complex structures of data with a convenient RDF-style output. 17 | 18 | *So what does this do again?* 19 | 20 | You write transactions to assert information about entities from a blank slate. 21 | Then datoms-differ finds out what's changed since the last time. It's a differ 22 | that takes the old data plus a datascript-like transaction of entity maps, and 23 | outputs a set of tx-data additions and retractions. 24 | 25 | *What the what?* 26 | 27 | Yeah, still a good question. 28 | 29 | ## Install 30 | 31 | Add `[datoms-differ "2025-11-14"]` to `:dependencies` in your `project.clj`. 32 | 33 | ## API 34 | 35 | Require `[datoms-differ.api]`. 36 | 37 | ### `(create-conn schema)` 38 | 39 | Takes a datascript schema and creates a "connection" (really, an atom with an 40 | empty db). A datascript schema might look like this: 41 | 42 | ``` 43 | (def schema 44 | {:route/number {:db/unique :db.unique/identity} 45 | :route/vessels {:db/valueType :db.type/ref 46 | :db/cardinality :db.cardinality/many} 47 | :vessel/imo {:db/unique :db.unique/identity}}) 48 | ``` 49 | 50 | Like datascript, datoms-differ only cares about `:db.unique/identity`, 51 | `:db.type/ref`, `:db.cardinality/many` and `:db/isComponent`. 52 | 53 | ### `(transact! conn source entity-maps)` 54 | 55 | Takes a connection, a source identifier and a list of entity maps, and 56 | transacts them into the connection. 57 | 58 | - `conn` - the atom you created with `create-conn` 59 | - `source` - you can put data into the connection from multiple sources, but 60 | since datoms-differ will retract missing values, it will only retract values 61 | previously asserted by the same source. 62 | - `entity-maps` - a list of entity maps that are asserted by this source. Example: 63 | 64 | ``` 65 | [{:route/number "100" 66 | :route/name "Stavanger-Tau" 67 | :route/vessels [{:vessel/imo "123" 68 | :vessel/mmsi "456"}]} 69 | {:vessel/imo "123" 70 | :vessel/name "MF Hardanger"}] 71 | ``` 72 | 73 | Note that every entity map needs to contain one and only one attribute that is 74 | marked as `:db.unique/identity` in the schema. 75 | 76 | Another interesting note about the above example is that since `:route/vessels` 77 | is marked as `:db.type/ref`, and `:vessel/imo` is `:db.unique/identity`, there 78 | will only be two entities as a result of this transaction. The route and one 79 | vessel with all three asserted attributes (imo, mmsi and name). 80 | 81 | The returned value from `transact!` has `:db-after`, `:db-before`, but most 82 | interestingly it has a `:tx-data` list of datoms. **This is the diff! Here it 83 | is!** 84 | 85 | ## Updating multiple sources at once 86 | 87 | If multiple sources has changed at the same time, you can use 88 | `transact-sources!` to assert all transactions at once. 89 | 90 | ### `(transact-sources! conn source->entity-maps)` 91 | 92 | Takes a connection and a map from source identifier to a list of entity maps, 93 | and transacts them all into the connection. 94 | 95 | In addition to being more efficient, and simpler to work with when you have 96 | multiple sources updating at once, this also avoids an issue where old data + 97 | partial new data would make a conflict, where the conflict is resolved in 98 | another part of the new data (not yet asserted). 99 | 100 | Note that all `source`s must be comparable with `compare`, since they are sorted 101 | in indices. 102 | 103 | ## Exporting to datascript 104 | 105 | There's also some tools for exporting to datascript. This lets you create a 106 | datascript db in the client, and then keep it up to date with new txes. 107 | 108 | Use `[datoms-differ.api]` with `(export-db db)` that gives you a string that 109 | can be read by clojurescript (when datascript is loaded) to create a datascript 110 | db. 111 | 112 | If you want to reduce the amount of bytes sent over the wire, you can also use 113 | `(prune-diffs schema tx-data)` to remove retractions of values that are later 114 | asserted. This optimalisation is possible since datascript doesn't have a notion 115 | of history. 116 | 117 | ## Contribute 118 | 119 | Yes, please do. And add tests for your feature or fix, or I'll 120 | certainly break it later. 121 | 122 | #### Running the tests 123 | 124 | `bin/kaocha` will run all tests. 125 | 126 | `bin/kaocha --watch` will run all the tests indefinitely. It sets up a 127 | watcher on the code files. If they change, only the relevant tests will be 128 | run again. 129 | 130 | ## License 131 | 132 | Copyright © (iterate inc 2017) Magnar Sveen 133 | 134 | Distributed under the Eclipse Public License, the same as Clojure. 135 | -------------------------------------------------------------------------------- /src/datoms_differ/datom.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.datom 2 | (:require [me.tonsky.persistent-sorted-set :as sset]) 3 | (:import (clojure.lang Seqable IPersistentCollection Indexed ILookup Associative MapEntry) 4 | (java.lang IllegalArgumentException Object UnsupportedOperationException))) 5 | 6 | (defn combine-hashes [x y] 7 | #?(:clj (clojure.lang.Util/hashCombine x y) 8 | :cljs (hash-combine x y))) 9 | 10 | (declare hash-datom equiv-datom seq-datom nth-datom assoc-datom val-at-datom) 11 | 12 | (deftype Datom [^int e a v s ^:unsynchronized-mutable ^int _hash] 13 | Object 14 | (hashCode [d] 15 | (if (zero? _hash) 16 | (let [h (int (hash-datom d))] 17 | (set! _hash h) 18 | h) 19 | _hash)) 20 | (toString [d] (pr-str d)) 21 | 22 | Seqable 23 | (seq [d] (seq-datom d)) 24 | 25 | IPersistentCollection 26 | (equiv [d o] (and (instance? Datom o) (equiv-datom d o))) 27 | (empty [_] (throw (UnsupportedOperationException. "empty is not supported on Datom"))) 28 | (count [_] 4) 29 | (cons [d [k v]] (assoc-datom d k v)) 30 | 31 | Indexed 32 | (nth [this i] (nth-datom this i)) 33 | (nth [this i not-found] (nth-datom this i not-found)) 34 | 35 | ILookup 36 | (valAt [d k] (val-at-datom d k nil)) 37 | (valAt [d k nf] (val-at-datom d k nf)) 38 | 39 | Associative 40 | (entryAt [d k] (when-let [v (val-at-datom d k nil)] 41 | (MapEntry. k v))) 42 | (containsKey [_ k] (#{:e :a :v :s} k)) 43 | (assoc [d k v] (assoc-datom d k v))) 44 | 45 | 46 | (defn datom ^Datom [e a v s] (Datom. e a v s 0)) 47 | 48 | 49 | (defn- hash-datom [^Datom d] 50 | (-> (hash (.-e d)) 51 | (combine-hashes (hash (.-a d))) 52 | (combine-hashes (hash (.-v d))) 53 | (combine-hashes (hash (.-s d))))) 54 | 55 | (defn- equiv-datom [^Datom d ^Datom o] 56 | (and (== (.-e d) (.-e o)) 57 | (= (.-a d) (.-a o)) 58 | (= (.-v d) (.-v o)) 59 | (= (.-s d) (.-s o)))) 60 | 61 | (defn- seq-datom [^Datom d] 62 | (list (.-e d) (.-a d) (.-v d) (.-s d))) 63 | 64 | (defn- val-at-datom [^Datom d k not-found] 65 | (case k 66 | :e (.-e d) 67 | :a (.-a d) 68 | :v (.-v d) 69 | :s (.-s d) 70 | not-found)) 71 | 72 | (defn- nth-datom 73 | ([^Datom d ^long i] 74 | (case i 75 | 0 (.-e d) 76 | 1 (.-a d) 77 | 2 (.-v d) 78 | 3 (.-s d) 79 | #?(:clj (throw (IndexOutOfBoundsException.)) 80 | :cljs (throw (js/Error. (str "Datom/-nth: Index out of bounds: " i)))))) 81 | ([^Datom d ^long i not-found] 82 | (case i 83 | 0 (.-e d) 84 | 1 (.-a d) 85 | 2 (.-v d) 86 | 3 (.-s d) 87 | not-found))) 88 | 89 | (defn- assoc-datom ^Datom [^Datom d k v] 90 | (case k 91 | :e (datom v (.-a d) (.-v d) (.-s d) ) 92 | :a (datom (.-e d) v (.-v d) (.-s d) ) 93 | :v (datom (.-e d) (.-a d) v (.-s d) ) 94 | :s (datom (.-e d) (.-a d) (.-v d) v ) 95 | (throw (IllegalArgumentException. (str "invalid key for Datom: " k))))) 96 | 97 | #?(:clj 98 | (defmethod print-method Datom [^Datom d, ^java.io.Writer w] 99 | (.write w (str "#differ/Datom ")) 100 | (binding [*out* w] 101 | (pr [(.-e d) (.-a d) (.-v d) (.-s d)])))) 102 | 103 | ;; Comparing datoms 104 | #?(:clj 105 | (defmacro combine-cmp [& comps] 106 | (loop [comps (reverse comps) 107 | res (num 0)] 108 | (if (not-empty comps) 109 | (recur 110 | (next comps) 111 | `(let [c# ~(first comps)] 112 | (if (== 0 c#) 113 | ~res 114 | c#))) 115 | res)))) 116 | 117 | (defprotocol DatomValueComparator 118 | "A protocol that allows you to provide custom compare for types of your choosing for the value attribute of a datom" 119 | (compare-value [this other])) 120 | 121 | (defn cmp [o1 o2] 122 | (if (nil? o1) 0 123 | (if (nil? o2) 0 124 | (compare o1 o2)))) 125 | 126 | (extend-protocol DatomValueComparator 127 | Object 128 | (compare-value [o1 o2] (cmp o1 o2)) 129 | 130 | nil 131 | (compare-value [_ _] 0)) 132 | 133 | (defn cmp-datoms-eavs [^Datom d1, ^Datom d2] 134 | (combine-cmp 135 | (#?(:clj Integer/compare :cljs -) (.-e d1) (.-e d2)) 136 | (cmp (.-a d1) (.-a d2)) 137 | (compare-value (.-v d1) (.-v d2)) 138 | (cmp (.-s d1) (.-s d2)))) 139 | 140 | (defn cmp-datoms-eav-only [^Datom d1, ^Datom d2] 141 | (combine-cmp 142 | (#?(:clj Integer/compare :cljs -) (.-e d1) (.-e d2)) 143 | (cmp (.-a d1) (.-a d2)) 144 | (compare-value (.-v d1) (.-v d2)))) 145 | 146 | (defn empty-eavs [] 147 | (sset/sorted-set-by cmp-datoms-eavs)) 148 | 149 | (defn to-eavs [datoms] 150 | (if (seq datoms) 151 | (sset/from-sequential cmp-datoms-eavs datoms) 152 | (empty-eavs))) 153 | 154 | (defn to-eav-only [datoms] 155 | (if (seq datoms) 156 | (->> datoms 157 | (reduce conj! (transient [])) 158 | persistent! 159 | (sset/from-sequential cmp-datoms-eav-only)) 160 | (empty-eavs))) 161 | 162 | (defn diff-in-value? [^Datom a ^Datom b] 163 | (and (= (.-e a) (.-e b)) 164 | (= (.-a a) (.-a b)) 165 | (not= (.-v a) (.-v b)))) 166 | 167 | (defn source-equals? [source ^Datom d] 168 | (= source (.s d))) 169 | 170 | (defn contains-eav? [eavs ^Datom d] 171 | (sset/slice eavs 172 | (datom (.-e d) (.-a d) (.-v d) nil) 173 | (datom (.-e d) (.-a d) (.-v d) nil))) 174 | -------------------------------------------------------------------------------- /dev/perf.clj: -------------------------------------------------------------------------------- 1 | (ns perf 2 | (:require [criterium.core :as crit] 3 | [datoms-differ.api :as c2] 4 | [datoms-differ.core :as c] 5 | [datoms-differ.datom :as d] 6 | [medley.core :ref [map-vals]] 7 | [me.tonsky.persistent-sorted-set.arrays :as arrays] 8 | [me.tonsky.persistent-sorted-set :as set] 9 | [taoensso.tufte :as tufte]) 10 | (:import [datoms_differ.datom Datom])) 11 | 12 | (comment 13 | (def history-schema 14 | {:vessel/imo {:spec string? :db/unique :db.unique/identity} 15 | :vessel/name {:spec string?} 16 | :vessel/type {} 17 | :vessel/dim-port {:spec int?} 18 | :vessel/dim-starboard {:spec int?} 19 | :vessel/dim-bow {:spec int?} 20 | :vessel/dim-stern {:spec int?} 21 | 22 | :position/lon {} 23 | :position/lat {} 24 | :position/inst {:db/unique :db.unique/identity :spec inst?} 25 | :position/speed {:spec number?} 26 | :position/heading {:spec int?} 27 | :position/rate-of-turn {:spec int?} 28 | :position/course {:spec number?} 29 | 30 | :entity/id {:spec string? :db/unique :db.unique/identity} 31 | :location/place-id {:spec string?} 32 | :location/lat {} 33 | :location/type {:spec keyword?} 34 | :location/lon {} 35 | :location/polygon {:spec vector? :compound-value? true} 36 | :location/name {:spec string?} 37 | 38 | :move/id {:spec string? :db/unique :db.unique/identity} 39 | :move/inst {:spec inst?} 40 | :move/imo {:spec string?} 41 | :move/location {:db/valueType :db.type/ref :db/cardinality :db.cardinality/one} 42 | :move/kind {:spec keyword?}}) 43 | 44 | ;; TODO: DOH, You'll have to try to get hold of your own test fixture for this for now (: 45 | (def history-entities 46 | (->> (slurp "/Users/mrundberget/projects/datoms-differ/history-entities.edn") 47 | (clojure.edn/read-string {}))) 48 | 49 | (def modified-moves 50 | {:prepare-moves [#:move{:id "9855783-:departure-1582710092606-new", :imo "9855783", :location #:entity{:id "location/haugesund-1"}, :kind :departure, :inst #inst "2020-02-26T09:41:32.606-00:00"} 51 | #:move{:id "9855783-:arrival-1582711626542-new", :imo "9855783", :location #:entity{:id "location/røvær-1"}, :kind :arrival, :inst #inst "2020-02-26T10:07:06.542-00:00"} 52 | #:move{:id "9855783-:departure-1582713035833-new", :imo "9855783", :location #:entity{:id "location/røvær-1"}, :kind :departure, :inst #inst "2020-02-26T10:30:35.833-00:00"}]}) 53 | 54 | 55 | ;; profiling 56 | (tufte/add-basic-println-handler! 57 | {:format-pstats-opts {:columns [:n-calls :p50 :p90 :mean :clock :total] 58 | :format-id-fn name}}) 59 | 60 | (tufte/profile {} (dotimes [i 50] 61 | (tufte/p :transact (transact-for-existing)))) 62 | 63 | 64 | (let [sample-db (c2/create-conn history-schema)] 65 | (println "\n******** NEW CLEAN DB *********") 66 | (time (c2/transact-sources! sample-db history-entities)) 67 | (println "\n******** NEW UPDATE ***********") 68 | (time (c2/transact-sources! sample-db history-entities)) 69 | (println "\n******** NEW TINY UPDATE ***********") 70 | (time (c2/transact-sources! sample-db modified-moves)) 71 | (count (:eavs @sample-db))) 72 | 73 | (let [sample-db (c/create-conn history-schema)] 74 | (println "\n******** OLD CLEAN DB ********") 75 | (time (c/transact-sources! sample-db history-entities)) 76 | (println "\n******** OLD UPDATE ***********") 77 | (time (c/transact-sources! sample-db history-entities)) 78 | (println "\n******** NEW TINY UPDATE ***********") 79 | (time (c/transact-sources! sample-db modified-moves)) 80 | nil) 81 | 82 | 83 | ;; ************* benchmarking ********************** 84 | 85 | 86 | ;; NEW IMPL TESTS 87 | (def first-history-db 88 | (let [sample-db (c2/create-conn history-schema)] 89 | (c2/transact-sources! sample-db history-entities) 90 | sample-db)) 91 | 92 | (defn transact-for-empty [] 93 | (c2/transact-sources! (c2/create-conn history-schema) history-entities)) 94 | 95 | (defn transact-for-existing [] 96 | (let [db (atom @first-history-db)] 97 | (c2/transact-sources! db history-entities))) 98 | 99 | (defn transact-for-existing-small-change [] 100 | (let [db (atom @first-history-db)] 101 | (c2/transact-sources! db modified-moves))) 102 | 103 | (def observations (->> (:prepare-observations history-entities) (drop 700) (take 100))) 104 | (defn transact-for-lots-off-add-and-remove [] 105 | (let [db (atom @first-history-db)] 106 | (c2/transact-sources! db {:prepare-observations observations}))) 107 | 108 | (crit/quick-bench (transact-for-empty)) 109 | (crit/quick-bench (transact-for-existing)) 110 | (crit/quick-bench (transact-for-existing-small-change)) 111 | (crit/quick-bench (transact-for-lots-off-add-and-remove)) 112 | 113 | (crit/with-progress-reporting (crit/bench (transact-for-empty))) 114 | (crit/with-progress-reporting (crit/bench (transact-for-existing))) 115 | (crit/with-progress-reporting (crit/bench (transact-for-existing-small-change))) 116 | 117 | 118 | ;; OLD IMPL TESTS 119 | (def first-history-db-old 120 | (let [sample-db (c/create-conn history-schema)] 121 | (c/transact-sources! sample-db history-entities) 122 | sample-db)) 123 | 124 | (defn transact-for-empty-old [] 125 | (c/transact-sources! (c/create-conn history-schema) history-entities)) 126 | 127 | (defn transact-for-existing-old [] 128 | (let [db (atom @first-history-db-old)] 129 | (c/transact-sources! db history-entities))) 130 | 131 | (defn transact-for-existing-old-small-change [] 132 | (let [db (atom @first-history-db-old)] 133 | (c/transact-sources! db modified-moves))) 134 | 135 | (crit/with-progress-reporting (crit/bench (transact-for-empty-old))) 136 | (crit/with-progress-reporting (crit/bench (transact-for-existing-old))) 137 | (crit/with-progress-reporting (crit/bench (transact-for-existing-old-small-change))) 138 | 139 | ) 140 | 141 | (comment 142 | (def sample-datoms 143 | [(d/datom 536870912 :vessel/imo "1234" :prepare-vessels) 144 | (d/datom 536870912 :vessel/name "Fjordnerd" :prepare-vessels) 145 | (d/datom 536870912 :vessel/type :mf :prepare-vessels) 146 | (d/datom 536870913 :vessel/imo "5678" :prepare-vessels) 147 | (d/datom 536870913 :vessel/name "Limasol" :prepare-vessels) 148 | (d/datom 536870913 :vessel/type :ms :prepare-vessels) 149 | (d/datom 536870912 :vessel/imo "1234" :prepare-observations) 150 | (d/datom 536870912 :vessel/lat 60 :prepare-observations) 151 | (d/datom 536870912 :vessel/lon 59 :prepare-observations) 152 | ;; conflict 153 | (d/datom 536870912 :vessel/imo "12345" :prepare-infos) 154 | ;; dupe 155 | (d/datom 536870912 :vessel/imo "1234" :prepare-vessels)]) 156 | 157 | (d/to-eavs sample-datoms) 158 | 159 | 160 | ;; TODO: Test custom compare using protocol 161 | (defn compare-map [x y] 162 | (if (map? y) 163 | (compare (hash x) (hash y)) 164 | (throw (Exception. (str "Cannot compare " x " to " y))))) 165 | 166 | (defn compare-set [x y] 167 | (if (set? y) 168 | (compare (hash x) (hash y)) 169 | (throw (Exception. (str "Cannot compare " x " to " y))))) 170 | 171 | (extend-protocol d/DatomValueComparator 172 | clojure.lang.PersistentArrayMap 173 | (customCompareTo [x y] (compare-map x y)) 174 | 175 | clojure.lang.PersistentTreeMap 176 | (customCompareTo [x y] (compare-map x y)) 177 | 178 | clojure.lang.PersistentHashMap 179 | (customCompareTo [x y] (compare-map x y)) 180 | 181 | clojure.lang.PersistentHashSet 182 | (customCompareTo [x y] (compare-set x y))) 183 | 184 | (d/cmp-datoms-eav-only (d/datom 1 :vessel/imo {:dill/dall #{1}} :dummy) 185 | (d/datom 1 :vessel/imo {:dill/dall #{}} :dammy)) 186 | 187 | 188 | ) 189 | -------------------------------------------------------------------------------- /src/datoms_differ/core.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.core 2 | "Deprecation warning: This namespace is here for backwards compatibility. You 3 | probably want to use datoms-differ.api instead." 4 | (:require [clojure.set :as set] 5 | [datoms-differ.impl.core-helpers :as ch])) 6 | 7 | (def default-db-id-partition 8 | {:from 0x10000000 9 | :to 0x1FFFFFFF}) 10 | 11 | (defn create-refs-lookup [{:keys [from to]} old-refs all-refs] 12 | (let [lowest-new-eid (inc (apply max 13 | (dec from) 14 | (filter #(<= from % to) (vals old-refs))))] 15 | (->> all-refs 16 | (remove old-refs) 17 | (map-indexed (fn [i ref] 18 | (if (= (first ref) :db/id) 19 | (let [eid (second ref)] 20 | (when (<= from eid to) 21 | (throw (ex-info "Asserted :db/id cannot be within the internal db-id-partition, check :datoms-differ.core/db-id-partition" {:ref ref :internal-partition {:from from :to to}}))) 22 | [ref (second ref)]) 23 | (let [eid (+ lowest-new-eid i)] 24 | (when-not (<= from eid to) 25 | (throw (ex-info "Generated internal eid falls outside internal db-id-partition, check :datoms-differ.core/db-id-partition" {:ref ref :eid eid :internal-partition {:from from :to to}}))) 26 | [ref eid])))) 27 | (into old-refs)))) 28 | 29 | (defn flatten-entity-map [{:keys [ref? many? component?] :as attrs} refs entity] 30 | (let [eid (refs (ch/get-entity-ref attrs entity)) 31 | disallow-nils (fn [k v] 32 | (when (nil? v) 33 | (throw (ex-info "Attributes cannot be nil" {:entity (ch/get-entity-ref attrs entity) 34 | :key k}))))] 35 | (mapcat (fn [[k v]] 36 | (cond 37 | (= k :db/id) 38 | nil ;; db/id is not an attribute so exclude it 39 | 40 | (ref? k) 41 | (doall 42 | (for [v (if (many? k) v [v])] 43 | (do 44 | (disallow-nils k v) 45 | [eid k (if (number? v) 46 | v 47 | (refs (ch/get-entity-ref attrs v)))]))) 48 | 49 | (ch/reverse-ref? k) 50 | (let [reverse-k (ch/reverse-ref-attr k)] 51 | (for [ref-entity-map (if (component? reverse-k) [v] v)] 52 | [(refs (ch/get-entity-ref attrs ref-entity-map)) reverse-k eid])) 53 | 54 | :else ;; scalar 55 | (doall 56 | (for [v (if (many? k) v [v])] 57 | (do 58 | (disallow-nils k v) 59 | [eid k v]))))) 60 | entity))) 61 | 62 | (defn disallow-conflicting-values [refs {:keys [many?]} datoms] 63 | (doseq [[[e a] datoms] (group-by #(take 2 %) datoms)] 64 | (when (and (not (many? a)) 65 | (< 1 (count datoms))) 66 | (throw (ex-info "Conflicting values asserted for entity" 67 | (let [e->entity-ref (set/map-invert refs)] 68 | {:entity-ref (e->entity-ref e) 69 | :attr a 70 | :conflicting-values (into #{} (map #(nth % 2) datoms))})))))) 71 | 72 | (defn disallow-empty-entities [all-entities datoms refs] 73 | (let [entity-id-has-datoms? (set (map first datoms)) 74 | entity-id-is-known-ref? (set (map second refs))] 75 | (doseq [e all-entities] 76 | (when (and (empty? (dissoc e :db/id)) 77 | (not (entity-id-has-datoms? (:db/id e))) 78 | (not (entity-id-is-known-ref? (:db/id e)))) 79 | (throw (ex-info (str "No attributes asserted for entity: " (pr-str e)) {})))))) 80 | 81 | (defn explode [{:keys [schema refs]} entity-maps] 82 | (let [attrs (ch/find-attrs schema) 83 | all-entities (ch/find-all-entities attrs entity-maps) 84 | entity-refs (distinct (map #(ch/get-entity-ref attrs %) all-entities)) 85 | new-refs (create-refs-lookup (::db-id-partition schema default-db-id-partition) refs entity-refs) 86 | datoms (set (mapcat #(flatten-entity-map attrs new-refs %) all-entities))] 87 | (disallow-conflicting-values new-refs attrs datoms) 88 | (disallow-empty-entities all-entities datoms refs) 89 | {:refs new-refs 90 | :datoms datoms})) 91 | 92 | (defn diff [datoms-before datoms-after] 93 | (let [new (set/difference datoms-after datoms-before) 94 | old (set/difference datoms-before datoms-after)] 95 | (concat 96 | (for [[e a v] old] [:db/retract e a v]) 97 | (for [[e a v] new] [:db/add e a v])))) 98 | 99 | (defn empty-db [schema] 100 | {:schema schema 101 | :refs {} 102 | :source-datoms {}}) 103 | 104 | (defn ^:export create-conn [schema] 105 | (atom (empty-db schema))) 106 | 107 | (defn get-datoms [db] 108 | (apply set/union #{} (vals (:source-datoms db)))) 109 | 110 | (defn disallow-conflicting-sources [db] 111 | (let [{:keys [many? ref?]} (ch/find-attrs (:schema db))] 112 | (doseq [[_ datoms] (->> (:source-datoms db) 113 | (mapcat (fn [[source datoms]] 114 | (keep (fn [[e a v]] 115 | (when-not (many? a) 116 | {:e e :a a :v v :source source})) 117 | datoms))) 118 | (group-by (fn [{:keys [e a]}] [e a])))] 119 | (when (< 1 (count (set (map :v datoms)))) 120 | (let [e->entity-ref (set/map-invert (:refs db))] 121 | (throw 122 | (ex-info "Conflicting values asserted between sources" 123 | (into {} 124 | (for [{:keys [e a v source]} datoms] 125 | [source [(e->entity-ref e) a (if (ref? a) 126 | (e->entity-ref v) 127 | v)]]))))))))) 128 | 129 | (defn- update-db-with-source-entity-maps [db source entity-maps] 130 | (let [{:keys [datoms refs]} (explode db entity-maps)] 131 | (if (-> entity-maps meta :partial-update?) 132 | (update-in (assoc db :refs refs) [:source-datoms source] #(into % datoms)) 133 | (assoc-in (assoc db :refs refs) [:source-datoms source] datoms)))) 134 | 135 | (defn with [db source entity-maps] 136 | (let [db-after (update-db-with-source-entity-maps db source entity-maps) 137 | _ (disallow-conflicting-sources db-after)] 138 | {:tx-data (diff (get-datoms db) (get-datoms db-after)) 139 | :db-before db 140 | :db-after db-after})) 141 | 142 | (defn with-sources [db source->entity-maps] 143 | (let [db-after (reduce (fn [db [source entity-maps]] 144 | (update-db-with-source-entity-maps db source entity-maps)) 145 | db 146 | source->entity-maps) 147 | _ (disallow-conflicting-sources db-after)] 148 | {:tx-data (diff (get-datoms db) (get-datoms db-after)) 149 | :db-before db 150 | :db-after db-after})) 151 | 152 | (defn ^:export transact! [conn source entity-maps] 153 | (let [report (atom nil)] 154 | (swap! conn (fn [db] 155 | (let [r (with db source entity-maps)] 156 | (reset! report r) 157 | (:db-after r)))) 158 | @report)) 159 | 160 | (defn ^:export transact-sources! [conn source->entity-maps] 161 | (let [report (atom nil)] 162 | (swap! conn (fn [db] 163 | (let [r (with-sources db source->entity-maps)] 164 | (reset! report r) 165 | (:db-after r)))) 166 | @report)) 167 | -------------------------------------------------------------------------------- /src/datoms_differ/reporter.clj: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.reporter 2 | (:require [clansi] 3 | [clojure.set :as set] 4 | [clojure.string :as str] 5 | [datoms-differ.impl.core-helpers :as ch])) 6 | 7 | (defn replace-entity-ids-with-identifier [schema datoms] 8 | (let [{:keys [ref? identity?]} (ch/find-attrs schema) 9 | reverse-id-lookup (into {} (keep (fn [[e a v]] 10 | (when (identity? a) 11 | [e [a v]])) datoms))] 12 | (for [[e a v] datoms] 13 | [(reverse-id-lookup e) 14 | a 15 | (if (ref? a) 16 | (reverse-id-lookup v) 17 | v)]))) 18 | 19 | (defn find-individual-changes [old-datoms new-datoms] 20 | (let [old-datoms (set old-datoms) 21 | new-datoms (set new-datoms) 22 | removed (set/difference (set old-datoms) (set new-datoms)) 23 | added (set/difference (set new-datoms) (set old-datoms)) 24 | changed (let [removed-ea->datoms (group-by (fn [[e a _]] [e a]) removed) 25 | added-ea->datoms (group-by (fn [[e a _]] [e a]) added) 26 | removed-and-added-eas (set/intersection (set (keys removed-ea->datoms)) 27 | (set (keys added-ea->datoms)))] 28 | (keep (fn [ea] 29 | (when (and (not (next (removed-ea->datoms ea))) 30 | (not (next (added-ea->datoms ea)))) ;; one removed, one added 31 | [(first (removed-ea->datoms ea)) 32 | (first (added-ea->datoms ea))])) 33 | removed-and-added-eas)) 34 | removed (set/difference removed (set (map first changed))) 35 | added (set/difference added (set (map second changed)))] 36 | (-> [] 37 | (into (for [datom removed] 38 | (into [:removed] datom))) 39 | (into (for [datom added] 40 | (into [:added] datom))) 41 | (into (for [[[e a old-v] [_ _ new-v]] changed] 42 | [:changed e a [old-v new-v]]))))) 43 | 44 | (defn max-by [k coll] 45 | (when (seq coll) 46 | (apply max-key k coll))) 47 | 48 | (defn group-changes [changes] 49 | (loop [result [] 50 | remaining-changes changes] 51 | (if (empty? remaining-changes) 52 | result 53 | (let [same-e (->> remaining-changes 54 | (group-by (fn [[_event e _a _v]] e)) 55 | (max-by (fn [[_ entries]] (count entries)))) 56 | same-event+a (->> remaining-changes 57 | (group-by (fn [[event _e a _v]] [event a])) 58 | (max-by (fn [[_ entries]] (count entries)))) 59 | num-same-e-entries (count (second same-e)) 60 | num-same-event+a-entries (count (second same-event+a))] 61 | (cond 62 | (= 1 num-same-e-entries num-same-event+a-entries) 63 | (into result remaining-changes) 64 | 65 | (> num-same-event+a-entries num-same-e-entries) 66 | (recur (conj result 67 | (let [[[event a] entries] same-event+a] 68 | [:several-entities event a (for [[_ e _ v] entries] [e v])])) 69 | (remove (set (second same-event+a)) remaining-changes)) 70 | 71 | :else 72 | (recur (conj result 73 | (let [[e entries] same-e] 74 | [:same-entity e (for [[event _ a v] entries] [event a v])])) 75 | (remove (set (second same-e)) remaining-changes))))))) 76 | 77 | (defn- summarize-entity [e datoms] 78 | (into {} (keep (fn [[e* a v]] 79 | (when (and (= e* e) (not= e [a v])) 80 | [a v])) 81 | datoms))) 82 | 83 | (defn find-entire-entity-changes [old-datoms new-datoms] 84 | (let [old-es (set (map first old-datoms)) 85 | new-es (set (map first new-datoms)) 86 | removed-es (set/difference old-es new-es) 87 | added-es (set/difference new-es old-es) 88 | removed-entities (for [e removed-es] [e (summarize-entity e old-datoms)]) 89 | added-entities (for [e added-es] [e (summarize-entity e new-datoms)]) 90 | changed-es (let [removed-vals->es (group-by second removed-entities)] 91 | (keep (fn [[e added-vals]] 92 | (when-let [es (removed-vals->es added-vals)] 93 | (when-not (next es) 94 | [(ffirst es) e]))) 95 | added-entities)) 96 | changed? (set (mapcat identity changed-es)) 97 | removed-entities (remove (fn [[e _summary]] (changed? e)) removed-entities) 98 | added-entities (remove (fn [[e _summary]] (changed? e)) added-entities)] 99 | (cond-> [] 100 | (seq removed-entities) (into (for [[k v] (group-by ffirst removed-entities)] 101 | [:removed-entities k v])) 102 | (seq added-entities) (into (for [[k v] (group-by ffirst added-entities)] 103 | [:added-entities k v])) 104 | (seq changed-es) (conj [:changed-identities changed-es])))) 105 | 106 | (defn find-changes [old-datoms new-datoms] 107 | (let [entire-entity-changes (find-entire-entity-changes old-datoms new-datoms) 108 | handled-old-entity-id? (set (mapcat (fn [[k t v]] 109 | (cond 110 | (= :removed-entities k) (map first v) 111 | (= :changed-identities k) (map first t) 112 | :else nil)) 113 | entire-entity-changes)) 114 | handled-new-entity-id? (set (mapcat (fn [[k t v]] 115 | (cond 116 | (= :added-entities k) (map first v) 117 | (= :changed-identities k) (map second t) 118 | :else nil)) 119 | entire-entity-changes))] 120 | (concat 121 | entire-entity-changes 122 | (group-changes (find-individual-changes 123 | (remove #(handled-old-entity-id? (first %)) old-datoms) 124 | (remove #(handled-new-entity-id? (first %)) new-datoms)))))) 125 | 126 | (defn- pr-data [data] 127 | (let [s (pr-str data)] 128 | (if (< 38 (bounded-count 39 s)) 129 | (str (subs s 0 35) "..." (last s)) 130 | s))) 131 | 132 | (defn summarize-entities [entities direction verb] 133 | (let [groups (group-by (juxt ffirst second) entities)] 134 | (if (< 4 (bounded-count 5 groups)) ;; too many, summarize instead of iterating 135 | (for [[e entities] (group-by ffirst entities)] 136 | (str direction " " (count entities) "× [" e "], e.g. " 137 | (let [[[_ id] v] (first entities)] 138 | (str verb " " (pr-data v) " for " (pr-str id))))) 139 | (for [[[e v] entities] groups] 140 | (let [num (count entities)] 141 | (str verb " " (pr-data v) " for " num "× [" e "]" 142 | (if (< num 4) ": " ", e.g. ") 143 | (str/join " " (take 3 (map (comp pr-str second first) entities))))))))) 144 | 145 | (defn create-report [changes] 146 | (for [[t & args] changes] 147 | (cond 148 | (= :changed t) (let [[e a [old-v new-v]] args] 149 | {:text (str "Changed " e " " a " to " (pr-data new-v)) 150 | :details [(str "was " (pr-data old-v))]}) 151 | (= :removed t) (let [[e a v] args] 152 | {:text (str "Removed " a " from " e) 153 | :details [(str "was " (pr-data v))]}) 154 | (= :added t) (let [[e a v] args] 155 | {:text (str "Added " a " " (pr-data v) " to " e)}) 156 | 157 | (= :changed-identities t) (let [[changes] args] 158 | {:text (str (count changes) " entities changed identity") 159 | :details (for [[before after] changes] 160 | (str before " to " after))}) 161 | 162 | (#{:added-entities :removed-entities} t) 163 | (let [verb (if (= :added-entities t) "Added" "Removed") 164 | [type entities] args] 165 | (if (next entities) 166 | (let [num (count entities)] 167 | {:text (str verb " " num "× [" type "] entities") 168 | :details (cond-> 169 | (vec (take 3 (for [[[_ id] vals] entities] 170 | (str (pr-data id) " " (pr-data vals))))) 171 | (< 3 num) (conj (str "... and " (- num 3) " more.")))}) 172 | {:text (str verb " entity " (ffirst entities)) 173 | :details (for [[k v] (second (first entities))] 174 | (str (pr-data k) " " (pr-data v)))})) 175 | 176 | (= :same-entity t) 177 | (let [[e changes] args 178 | types (set (map first changes)) 179 | all-same-type? (= 1 (count types))] 180 | {:text (if all-same-type? 181 | (case (first types) 182 | :added (str "Added " (count changes) " attributes to " e) 183 | :removed (str "Removed " (count changes) " attributes from " e) 184 | :changed (str "Changed " (count changes) " attributes for " e)) 185 | (str "Changed " (count changes) " attributes for " e)) 186 | :details (for [[event a v] changes] 187 | (cond 188 | (= :added event) (str (when-not all-same-type? "added ") a " " (pr-data v)) 189 | (= :removed event) (str (when-not all-same-type? "removed ") a " " (pr-data v)) 190 | (= :changed event) (str "changed " a " to " (pr-data (second v)))))}) 191 | 192 | (= :several-entities t) 193 | (let [[t a entities] args] 194 | (cond 195 | (= :removed t) 196 | {:text (str "Removed " a " from " (count entities) " entities") 197 | :details (summarize-entities entities "from" "was")} 198 | 199 | (= :added t) 200 | {:text (str "Added " a " to " (count entities) " entities") 201 | :details (summarize-entities entities "to" "is")} 202 | 203 | (= :changed t) 204 | {:text (str "Changed " a " for " (count entities) " entities") 205 | :details (summarize-entities entities "for" "replaced")}))))) 206 | 207 | (defn ^:export render-report! [report] 208 | (doseq [{:keys [text details]} report] 209 | (println (clansi/style 210 | text 211 | (cond 212 | (str/starts-with? text "Added ") :green 213 | (str/starts-with? text "Removed ") :red 214 | :else :cyan))) 215 | (doseq [detail details] 216 | (println "-" detail)) 217 | (println " "))) 218 | -------------------------------------------------------------------------------- /src/datoms_differ/api.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.api 2 | (:require [clojure.set :as set] 3 | [datoms-differ.datom :as d] 4 | [datoms-differ.export :as dd-export] 5 | [datoms-differ.impl.core-helpers :as ch] 6 | [me.tonsky.persistent-sorted-set :as sset] 7 | [medley.core :refer [map-vals]]) 8 | (:import [datoms_differ.datom Datom])) 9 | 10 | (defn- diff-sorted [a b cmp] 11 | (loop [only-a (transient []) 12 | only-b (transient []) 13 | a a 14 | b b] 15 | (cond 16 | (empty? a) [(persistent! only-a) (into (persistent! only-b) b)] 17 | (empty? b) [(into (persistent! only-a) a) (persistent! only-b)] 18 | :else 19 | (let [first-a (first a) 20 | first-b (first b) 21 | diff (cmp first-a first-b)] 22 | (cond 23 | (== diff 0) (recur only-a only-b (next a) (next b)) 24 | (< diff 0) (recur (conj! only-a first-a) only-b (next a) b) 25 | (> diff 0) (recur only-a (conj! only-b first-b) a (next b))))))) 26 | 27 | (def default-db-id-partition 28 | {:from 0x10000000 29 | :to 0x1FFFFFFF}) 30 | 31 | (defn- get-lowest-new-eid [{:keys [from _to]} eavs] 32 | (inc (max 33 | (dec from) 34 | (or (-> eavs (sset/rslice nil nil) first :e) 0)))) 35 | 36 | (defn- create-refs-lookup [{:keys [schema attrs eavs refs]} entities] 37 | (let [{:keys [from to] :as db-id-partition} (::db-id-partition schema default-db-id-partition) 38 | lowest-new-eid (get-lowest-new-eid db-id-partition eavs)] 39 | (loop [idx lowest-new-eid 40 | new-refs (transient refs) 41 | entities-with-eid (transient []) 42 | entities entities] 43 | (let [entity (first entities) 44 | rf (when entity (ch/get-entity-ref attrs entity)) 45 | eid (when rf (new-refs rf))] 46 | (cond 47 | (nil? entity) [(persistent! new-refs) (persistent! entities-with-eid)] 48 | 49 | eid (recur idx 50 | new-refs 51 | (conj! entities-with-eid [entity eid]) 52 | (next entities)) 53 | 54 | :else 55 | (do 56 | (when-not (<= from idx to) 57 | (throw (ex-info "Generated internal eid falls outside internal db-id-partition, check :datoms-differ.core2/db-id-partition" 58 | {:ref rf :eid idx :internal-partition {:from from :to to}}))) 59 | (recur (inc idx) 60 | (assoc! new-refs rf idx) 61 | (conj! entities-with-eid [entity idx]) 62 | (next entities)))))))) 63 | 64 | (defn- flatten-all-entities [source {:keys [ref? many? component?] :as attrs} refs all-entities] 65 | (let [disallow-nils (fn [k v entity] 66 | (when (nil? v) 67 | (throw (ex-info "Attributes cannot be nil" {:entity (ch/get-entity-ref attrs entity) 68 | :key k})))) 69 | get-eid #(refs (ch/get-entity-ref-unsafe attrs %))] 70 | (->> all-entities 71 | (reduce 72 | (fn [acc [entity eid]] 73 | (reduce-kv 74 | (fn [acc k v] 75 | (cond 76 | (ref? k) 77 | (reduce (fn [acc v] 78 | (disallow-nils k v entity) 79 | (conj! acc (d/datom eid 80 | k 81 | (if (number? v) v (get-eid (ch/add-tuple-attributes attrs v))) 82 | source))) 83 | acc 84 | (if (many? k) v [v])) 85 | 86 | (ch/reverse-ref? k) 87 | (let [reverse-k (ch/reverse-ref-attr k)] 88 | (reduce (fn [acc ref-entity-map] 89 | (conj! acc (d/datom (get-eid (ch/add-tuple-attributes attrs ref-entity-map)) 90 | reverse-k 91 | eid 92 | source))) 93 | acc 94 | (if (component? reverse-k) [v] v))) 95 | 96 | :else ;; scalar 97 | (reduce (fn [acc v] 98 | (disallow-nils k v entity) 99 | (conj! acc (d/datom eid k v source))) 100 | acc 101 | (if (many? k) v [v])))) 102 | acc 103 | entity)) 104 | (transient [])) 105 | persistent! 106 | d/to-eavs))) 107 | 108 | (defn- explode-entity-maps [source {:keys [attrs] :as db} entity-maps] 109 | (let [all-entities (->> entity-maps 110 | (ch/find-all-entities attrs) 111 | (map (partial ch/add-tuple-attributes attrs))) 112 | [new-refs all-entities-with-eid] (create-refs-lookup db all-entities) 113 | datoms (flatten-all-entities source attrs new-refs all-entities-with-eid)] 114 | {:refs new-refs 115 | :datoms datoms})) 116 | 117 | (defn- find-conflicting-value [many? eavs] 118 | (:conflict 119 | (persistent! 120 | (reduce 121 | (fn [{:keys [prev] :as acc} ^Datom curr] 122 | (cond 123 | (many? (.-a curr)) 124 | acc 125 | 126 | (nil? prev) 127 | (assoc! acc :prev curr) 128 | 129 | (d/diff-in-value? prev curr) 130 | (reduced (assoc! acc :conflict [(:e curr) (:a curr)])) 131 | 132 | :else (assoc! acc :prev curr))) 133 | (transient {}) 134 | eavs)))) 135 | 136 | (defn- find-conflicting-value-by-additions [many? eavs to-add] 137 | (reduce 138 | (fn [acc ^Datom d] 139 | (if (many? (.-a d)) 140 | acc 141 | (let [search-d (d/datom (.-e d) (.-a d) nil nil) 142 | datoms (sset/slice eavs search-d search-d)] 143 | (if (find-conflicting-value many? datoms) 144 | (reduced [(.-e d) (.-a d)]) 145 | acc)))) 146 | nil 147 | to-add)) 148 | 149 | (defn- throw-conflicting-values 150 | [{:keys [attrs eavs refs]} [e a]] 151 | (let [{:keys [ref?]} attrs 152 | e->entity-ref (set/map-invert refs) 153 | datoms (sset/slice eavs (d/datom e a nil nil) (d/datom e a nil nil)) 154 | v-fn (if (ref? a) (comp e->entity-ref :v) :v)] 155 | (throw 156 | (ex-info "Conflicting values asserted for entity" 157 | {:attr a 158 | :entity-ref (e->entity-ref e) 159 | :conflict (->> datoms 160 | (group-by :s) 161 | (map-vals #(->> % (map v-fn) set)))})))) 162 | 163 | (defn- disallow-conflicting-values 164 | "Checks for conflicting values for all [e a]'s in db. Both across sources and for any given source. 165 | If a conflict is found it throws with info about source(s) and which values are conflicting." 166 | [{:keys [added eavs attrs] :as db}] 167 | (let [{:keys [many?]} attrs] 168 | ;; When there are few additions compared to total number of additions it's faster to just check the addition datoms 169 | (some->> (if (> (/ 5 100) (/ (count added) (max (count eavs) 1))) 170 | (find-conflicting-value-by-additions many? eavs added) 171 | (find-conflicting-value many? eavs)) 172 | (throw-conflicting-values db)))) 173 | 174 | (defn- find-source-diffs [source eavs datoms] 175 | (diff-sorted (filter (partial d/source-equals? source) eavs) 176 | datoms 177 | d/cmp-datoms-eav-only)) 178 | 179 | (defn- union! [s1 v2] 180 | (if (< (count s1) (count v2)) 181 | (reduce conj! (transient (d/to-eavs v2)) s1) 182 | (reduce conj! s1 v2))) 183 | 184 | (defn- update-eavs-by-diff 185 | "Fast path for updating the eavs index based on the results from a diff" 186 | [eavs [retracted added]] 187 | (loop [eavs-t (transient eavs) 188 | retracted retracted 189 | added added] 190 | (let [r (first retracted) 191 | a (first added)] 192 | (cond 193 | (and r a) (recur (-> eavs-t (disj! r) (conj! a)) (next retracted) (next added)) 194 | (and (nil? r) a) (persistent! (union! eavs-t added)) 195 | (and r (nil? a)) (persistent! (reduce disj! eavs-t retracted)) 196 | :else (persistent! eavs-t))))) 197 | 198 | (defn- prune-refs 199 | "Remove refs that are no longer present in eavs." 200 | [{:keys [attrs retracted eavs refs]}] 201 | (let [{:keys [identity?]} attrs] 202 | (cond 203 | (= 0 (count retracted)) 204 | refs 205 | 206 | (> (/ 1 6) (/ (count retracted) (max (count eavs) 1))) 207 | (persistent! 208 | (reduce (fn [acc [e a v]] 209 | (if (and (identity? a) 210 | (not (sset/slice eavs (d/datom e a nil nil) (d/datom e a nil nil)))) 211 | (dissoc! acc [a v]) 212 | acc)) 213 | (transient refs) 214 | retracted)) 215 | 216 | :else 217 | (persistent! 218 | (reduce (fn [acc ^Datom d] 219 | (if (identity? (.-a d)) 220 | (assoc! acc [(.-a d) (.-v d)] (.-e d)) 221 | acc)) 222 | (transient {}) 223 | eavs))))) 224 | 225 | (defn- create-tx-data 226 | "Creates datomic transaction data from sorted sequences of add and removed calculated for each source 227 | Due to potential presence of same datom present in multiple sources, 228 | some additional filtering is needed prior to generating the final tx report data" 229 | [{:keys [retracted added eavs]}] 230 | (concat 231 | (persistent! 232 | (reduce (fn [acc ^Datom d] 233 | (if (d/contains-eav? eavs d) 234 | acc 235 | (conj! acc [:db/retract (.-e d) (.-a d) (.-v d)]))) 236 | (transient []) 237 | retracted)) 238 | (for [^Datom d (d/to-eav-only added)] 239 | [:db/add (.-e d) (.-a d) (.-v d)]))) 240 | 241 | (defn with-sources [db source->entity-maps] 242 | (let [attrs (ch/find-attrs (:schema db)) 243 | update-refs #(assoc % :refs (prune-refs %)) 244 | db-after (->> source->entity-maps 245 | (reduce (fn [db [source entity-maps]] 246 | (let [{:keys [datoms refs]} (explode-entity-maps source db entity-maps) 247 | [retracted added] (find-source-diffs source (:eavs db) datoms)] 248 | (-> db 249 | (update :retracted into retracted) 250 | (update :added into added) 251 | (assoc :refs refs) 252 | (update :eavs update-eavs-by-diff [retracted added])))) 253 | (assoc db :retracted [] :added [] :attrs attrs)) 254 | update-refs) 255 | _ (disallow-conflicting-values db-after)] 256 | {:tx-data (create-tx-data db-after) 257 | :db-before db 258 | :db-after (dissoc db-after :attrs :added :retracted)})) 259 | 260 | (defn with [db source entity-maps] 261 | (with-sources db {source entity-maps})) 262 | 263 | (defn- empty-db [schema] 264 | (when-not (map? schema) 265 | (throw (ex-info "Expected schema to be a map." {:type (type schema)}))) 266 | {:schema schema 267 | :refs {} 268 | :eavs (d/empty-eavs)}) 269 | 270 | (defn create-conn 271 | "Takes a datascript schema and creates a 'connection' (really, an atom with an empty db)" 272 | [schema] 273 | (atom (empty-db schema))) 274 | 275 | (defn transact-sources! 276 | "Takes a connection and a map from source identifier to a list of entity maps, and transacts them all into the connection." 277 | [conn source->entity-maps] 278 | (let [report (atom nil)] 279 | (swap! conn (fn [db] 280 | (let [r (with-sources db source->entity-maps)] 281 | (reset! report r) 282 | (:db-after r)))) 283 | @report)) 284 | 285 | (defn transact! 286 | "Takes a connection, a source identifier and a list of entity maps, and transacts them into the connection." 287 | [conn source entity-maps] 288 | (transact-sources! conn {source entity-maps})) 289 | 290 | ;; Convenience functions 291 | 292 | (defn explode 293 | "Given a schema and a list of entity-maps, you get a map of datoms (with generated entity ids) and a map of lookup-refs->eid back" 294 | [schema entity-maps] 295 | (let [{:keys [many? ref?] :as attrs} (ch/find-attrs schema) 296 | {:keys [refs datoms] :as res} (explode-entity-maps ::ignore 297 | {:schema schema 298 | :attrs attrs 299 | :eavs (d/empty-eavs) 300 | :refs {}} 301 | entity-maps)] 302 | 303 | (when-let [[e a] (find-conflicting-value many? datoms)] 304 | (let [e->entity-ref (set/map-invert refs) 305 | conflicting-datoms (sset/slice datoms (d/datom e a nil nil) (d/datom e a nil nil)) 306 | v-fn (if (ref? a) (comp e->entity-ref :v) :v)] 307 | (throw (ex-info "Conflicting values asserted for entity" 308 | {:entity-ref (e->entity-ref e) 309 | :attr a 310 | :conflicting-values (into #{} (map v-fn conflicting-datoms))})))) 311 | 312 | (update res :datoms #(map (fn [[e a v]] [e a v]) %)))) 313 | 314 | (defn get-datoms 315 | "Get all datoms `[e a v]` in given database (across sources)" 316 | [db] 317 | (->> (:eavs db) 318 | (d/to-eav-only) 319 | (map (fn [^Datom d] [(.-e d) (.-a d) (.-v d)])))) 320 | 321 | (defn export-db 322 | "Export database to DataScript. Gives you a string that can be read by clojurescript (when datascript is loaded) to create a datascript db." 323 | [{:keys [schema] :as db}] 324 | (dd-export/export (dd-export/prep-for-datascript schema) 325 | (get-datoms db) 326 | :start-tx (inc (:to default-db-id-partition)) 327 | :partition-key ::db-id-partition)) 328 | 329 | (def 330 | ^{:doc "Remove retractions of values that are later asserted in tx-data" 331 | :arglists '([schema tx-data])} 332 | prune-diffs dd-export/prune-diffs) 333 | -------------------------------------------------------------------------------- /test/datoms_differ/reporter_test.clj: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.reporter-test 2 | (:require [clojure.test :refer [deftest is testing]] 3 | [datoms-differ.core-test :refer [schema]] 4 | [datoms-differ.reporter :as sut])) 5 | 6 | (deftest replace-entity-ids-with-identifier 7 | (is (= (sut/replace-entity-ids-with-identifier schema [[1024 :route/number "100"] 8 | [1024 :route/name "Stavanger-Tau"] 9 | [1024 :route/services 1025] 10 | [1025 :service/id :s567]]) 11 | [[[:route/number "100"] :route/number "100"] 12 | [[:route/number "100"] :route/name "Stavanger-Tau"] 13 | [[:route/number "100"] :route/services [:service/id :s567]] 14 | [[:service/id :s567] :service/id :s567]]))) 15 | 16 | (deftest find-individual-changes 17 | (is (= (sut/find-individual-changes 18 | [[[:route/number "100"] :route/number "100"] 19 | [[:route/number "100"] :route/name "Stavanger-Tau"]] 20 | [[[:route/number "100"] :route/number "100"]]) 21 | [[:removed [:route/number "100"] :route/name "Stavanger-Tau"]])) 22 | 23 | (is (= (sut/find-individual-changes 24 | [[[:route/number "100"] :route/number "100"]] 25 | [[[:route/number "100"] :route/number "100"] 26 | [[:route/number "100"] :route/name "Stavanger-Tau"]]) 27 | [[:added [:route/number "100"] :route/name "Stavanger-Tau"]])) 28 | 29 | (is (= (sut/find-individual-changes 30 | [[[:route/number "100"] :route/services :service-1] 31 | [[:route/number "100"] :route/services :service-2] 32 | [[:route/number "100"] :route/services :service-3]] 33 | [[[:route/number "100"] :route/services :service-1]]) 34 | [[:removed [:route/number "100"] :route/services :service-3] 35 | [:removed [:route/number "100"] :route/services :service-2]])) 36 | 37 | (is (= (sut/find-individual-changes 38 | [[[:route/number "100"] :route/number "100"] 39 | [[:route/number "100"] :route/name "Stavanger-Tau"]] 40 | [[[:route/number "100"] :route/number "100"] 41 | [[:route/number "100"] :route/name "Stavanger - Tau"]]) 42 | [[:changed [:route/number "100"] :route/name ["Stavanger-Tau" "Stavanger - Tau"]]]))) 43 | 44 | (deftest group-changes 45 | (is (= (sut/group-changes 46 | [[:removed [:route/number "100"] :route/active? true] 47 | [:removed [:route/number "200"] :route/active? true] 48 | [:removed [:route/number "300"] :route/active? true] 49 | [:removed [:route/number "400"] :route/active? true]]) 50 | [[:several-entities :removed :route/active? [[[:route/number "100"] true] 51 | [[:route/number "200"] true] 52 | [[:route/number "300"] true] 53 | [[:route/number "400"] true]]]])) 54 | 55 | (is (= (sut/group-changes 56 | [[:changed [:route/number "100"] :route/name ["Stavanger-Tau" "Stavanger - Tau"]] 57 | [:removed [:route/number "100"] :route/active? true] 58 | [:added [:route/number "100"] :route/disabled? false]]) 59 | [[:same-entity [:route/number "100"] [[:changed :route/name ["Stavanger-Tau" "Stavanger - Tau"]] 60 | [:removed :route/active? true] 61 | [:added :route/disabled? false]]]])) 62 | 63 | (testing "the largest grouping is chosen" 64 | (is (= (sut/group-changes 65 | [[:changed [:route/number "100"] :route/name ["Stavanger-Tau" "Stavanger - Tau"]] 66 | [:removed [:route/number "100"] :route/active? true] 67 | [:removed [:route/number "200"] :route/active? true] 68 | [:removed [:route/number "300"] :route/active? true] 69 | [:removed [:route/number "400"] :route/active? true]]) 70 | [[:several-entities :removed :route/active? [[[:route/number "100"] true] 71 | [[:route/number "200"] true] 72 | [[:route/number "300"] true] 73 | [[:route/number "400"] true]]] 74 | [:changed [:route/number "100"] :route/name ["Stavanger-Tau" "Stavanger - Tau"]]])) 75 | 76 | (is (= (sut/group-changes 77 | [[:changed [:route/number "100"] :route/name ["Stavanger-Tau" "Stavanger - Tau"]] 78 | [:removed [:route/number "100"] :route/active? true] 79 | [:added [:route/number "100"] :route/disabled? false] 80 | [:removed [:route/number "200"] :route/active? true]]) 81 | [[:same-entity [:route/number "100"] [[:changed :route/name ["Stavanger-Tau" "Stavanger - Tau"]] 82 | [:removed :route/active? true] 83 | [:added :route/disabled? false]]] 84 | [:removed [:route/number "200"] :route/active? true]])) 85 | 86 | (is (= (sut/group-changes 87 | [[:changed [:route/number "100"] :route/name ["Stavanger-Tau" "Stavanger - Tau"]] 88 | [:added [:route/number "100"] :route/disabled? false] 89 | [:removed [:route/number "100"] :route/active? true] 90 | [:removed [:route/number "200"] :route/active? true] 91 | [:added [:route/number "200"] :route/disabled? false] 92 | [:removed [:route/number "300"] :route/active? true] 93 | [:removed [:route/number "400"] :route/active? true]]) 94 | [[:several-entities :removed :route/active? [[[:route/number "100"] true] 95 | [[:route/number "200"] true] 96 | [[:route/number "300"] true] 97 | [[:route/number "400"] true]]] 98 | [:same-entity [:route/number "100"] [[:changed :route/name ["Stavanger-Tau" "Stavanger - Tau"]] 99 | [:added :route/disabled? false]]] 100 | [:added [:route/number "200"] :route/disabled? false]])))) 101 | 102 | (deftest find-entire-entity-changes 103 | (is (= (sut/find-entire-entity-changes 104 | [[[:route/number "100"] :route/number "100"] 105 | [[:route/number "100"] :route/name "Stavanger-Tau"]] 106 | [[[:service/id :service123] :service/id :service123] 107 | [[:service/id :service123] :service/label "II"] 108 | [[:route/number "101"] :route/number "101"] 109 | [[:route/number "101"] :route/name "Fogn-Judaberg"]]) 110 | [[:removed-entities :route/number [[[:route/number "100"] {:route/name "Stavanger-Tau"}]]] 111 | [:added-entities :route/number [[[:route/number "101"] {:route/name "Fogn-Judaberg"}]]] 112 | [:added-entities :service/id [[[:service/id :service123] {:service/label "II"}]]]])) 113 | 114 | (is (= (sut/find-entire-entity-changes 115 | [[[:route/number "100"] :route/number "100"] 116 | [[:route/number "100"] :route/name "Stavanger-Tau"]] 117 | [[[:route/number "101"] :route/number "101"] 118 | [[:route/number "101"] :route/name "Stavanger-Tau"]]) 119 | [[:changed-identities [[[:route/number "100"] [:route/number "101"]]]]]))) 120 | 121 | (deftest find-changes 122 | (is (= (sut/find-changes 123 | [[[:route/number "100"] :route/number "100"] 124 | [[:route/number "100"] :route/name "Stavanger-Tau"] 125 | [[:route/number "200"] :route/number "200"] 126 | [[:route/number "200"] :route/name "Fogn-Judaberg"] 127 | [[:route/number "300"] :route/number "300"] 128 | [[:route/number "300"] :route/active? true]] 129 | [[[:route/number "101"] :route/number "101"] 130 | [[:route/number "101"] :route/name "Stavanger-Tau"] 131 | [[:route/number "200"] :route/number "200"] 132 | [[:route/number "200"] :route/name "Fogn-Jelsa-Judaberg"] 133 | [[:route/number "300"] :route/number "300"] 134 | [[:route/number "300"] :route/disabled? false] 135 | [[:service/id :service123] :service/id :service123] 136 | [[:service/id :service123] :service/label "II"]]) 137 | [[:added-entities :service/id [[[:service/id :service123] #:service{:label "II"}]]] 138 | [:changed-identities [[[:route/number "100"] [:route/number "101"]]]] 139 | [:same-entity [:route/number "300"] [[:removed :route/active? true] [:added :route/disabled? false]]] 140 | [:changed [:route/number "200"] :route/name ["Fogn-Judaberg" "Fogn-Jelsa-Judaberg"]]]))) 141 | 142 | (deftest create-report 143 | (is (= (sut/create-report [[:changed [:route/number "200"] :route/name ["Fogn-Judaberg" "Fogn-Jelsa-Judaberg"]]]) 144 | [{:text "Changed [:route/number \"200\"] :route/name to \"Fogn-Jelsa-Judaberg\"" 145 | :details ["was \"Fogn-Judaberg\""]}])) 146 | 147 | (is (= (sut/create-report [[:changed [:route/number "200"] :route/numbers [(range 10) (range 100)]]]) 148 | [{:text "Changed [:route/number \"200\"] :route/numbers to (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14...)" 149 | :details ["was (0 1 2 3 4 5 6 7 8 9)"]}])) 150 | 151 | (is (= (sut/create-report [[:removed [:route/number "100"] :route/name "Stavanger-Tau"]]) 152 | [{:text "Removed :route/name from [:route/number \"100\"]" 153 | :details ["was \"Stavanger-Tau\""]}])) 154 | 155 | (is (= (sut/create-report [[:added [:route/number "100"] :route/name "Stavanger-Tau"]]) 156 | [{:text "Added :route/name \"Stavanger-Tau\" to [:route/number \"100\"]"}])) 157 | 158 | (is (= (sut/create-report [[:several-entities :removed :route/active? [[[:route/number "100"] true] 159 | [[:route/number "200"] true] 160 | [[:route/number "300"] false] 161 | [[:route/number "400"] false] 162 | [[:route/number "500"] false] 163 | [[:route/number "600"] false]]]]) 164 | [{:text "Removed :route/active? from 6 entities" 165 | :details ["was true for 2× [:route/number]: \"100\" \"200\"" 166 | "was false for 4× [:route/number], e.g. \"300\" \"400\" \"500\""]}])) 167 | 168 | (is (= (sut/create-report [[:several-entities :added :some/details [[[:route/number "100"] 1] 169 | [[:route/number "200"] 2] 170 | [[:route/number "300"] 3] 171 | [[:route/number "400"] 4] 172 | [[:service/id :service-1] 5] 173 | [[:service/id :service-2] 6] 174 | [[:service/id :service-3] 7]]]]) 175 | [{:text "Added :some/details to 7 entities" 176 | :details ["to 4× [:route/number], e.g. is 1 for \"100\"" 177 | "to 3× [:service/id], e.g. is 5 for :service-1"]}])) 178 | 179 | (is (= (sut/create-report [[:several-entities :changed :route/active? [[[:route/number "100"] [true false]] 180 | [[:route/number "200"] [true false]] 181 | [[:route/number "300"] [true false]] 182 | [[:route/number "400"] [true false]] 183 | [[:route/number "500"] [true false]] 184 | [[:route/number "600"] [true false]]]]]) 185 | [{:text "Changed :route/active? for 6 entities" 186 | :details ["replaced [true false] for 6× [:route/number], e.g. \"100\" \"200\" \"300\""]}])) 187 | 188 | (is (= (sut/create-report [[:same-entity [:route/number "300"] [[:removed :route/active? true] 189 | [:changed :route/disabled? [false true]] 190 | [:added :route/details {:abcdefghi 123456 :defghijkl 456789 :ghijklmno 789123}]]]]) 191 | [{:text "Changed 3 attributes for [:route/number \"300\"]" 192 | :details ["removed :route/active? true" 193 | "changed :route/disabled? to true" 194 | "added :route/details {:abcdefghi 123456, :defghijkl 4567...}"]}])) 195 | 196 | (is (= (sut/create-report [[:same-entity [:route/number "300"] [[:removed :route/active? true] 197 | [:removed :route/disabled? false]]]]) 198 | [{:text "Removed 2 attributes from [:route/number \"300\"]" 199 | :details [":route/active? true" 200 | ":route/disabled? false"]}])) 201 | 202 | (is (= (sut/create-report [[:changed-identities [[[:route/number 100] [:route/number "100"]] 203 | [[:route/number 101] [:route/number "101"]]]]]) 204 | [{:text "2 entities changed identity" 205 | :details ["[:route/number 100] to [:route/number \"100\"]" 206 | "[:route/number 101] to [:route/number \"101\"]"]}])) 207 | 208 | (is (= (sut/create-report [[:added-entities :service/id [[[:service/id :service123] #:service{:label "II" :name "Foobar"}]]]]) 209 | [{:text "Added entity [:service/id :service123]" 210 | :details [":service/label \"II\"" 211 | ":service/name \"Foobar\""]}])) 212 | 213 | (is (= (sut/create-report [[:added-entities :service/id [[[:service/id :service123] #:service{:label "I" :name "Foobar"}] 214 | [[:service/id :service456] #:service{:label "II"}]]]]) 215 | [{:text "Added 2× [:service/id] entities" 216 | :details [":service123 #:service{:label \"I\", :name \"Foobar\"}" 217 | ":service456 #:service{:label \"II\"}"]}])) 218 | 219 | (is (= (sut/create-report [[:added-entities :route/number [[[:route/number 1] #:route{:name "One"}] 220 | [[:route/number 2] #:route{:name "Two"}] 221 | [[:route/number 3] #:route{:name "Three"}] 222 | [[:route/number 4] #:route{:name "Four"}] 223 | [[:route/number 5] #:route{:name "Five"}] 224 | [[:route/number 6] #:route{:name "Six"}]]]]) 225 | [{:text "Added 6× [:route/number] entities" 226 | :details ["1 #:route{:name \"One\"}" 227 | "2 #:route{:name \"Two\"}" 228 | "3 #:route{:name \"Three\"}" 229 | "... and 3 more."]}])) 230 | 231 | (is (= (sut/create-report [[:removed-entities :route/number [[[:route/number 1] #:route{:name "One"}] 232 | [[:route/number 2] #:route{:name "Two"}] 233 | [[:route/number 3] #:route{:name "Three"}] 234 | [[:route/number 4] #:route{:name "Four"}] 235 | [[:route/number 5] #:route{:name "Five"}] 236 | [[:route/number 6] #:route{:name "Six"}]]]]) 237 | [{:text "Removed 6× [:route/number] entities" 238 | :details ["1 #:route{:name \"One\"}" 239 | "2 #:route{:name \"Two\"}" 240 | "3 #:route{:name \"Three\"}" 241 | "... and 3 more."]}]))) 242 | -------------------------------------------------------------------------------- /test/datoms_differ/core_test.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.core-test 2 | (:require [clojure.test :refer [deftest is testing]] 3 | [datoms-differ.core :as sut] 4 | [datoms-differ.impl.core-helpers :as ch])) 5 | 6 | (def schema 7 | {:datoms-differ.core/db-id-partition {:from 1024 :to 2048} 8 | :route/name {} 9 | :route/number {:db/unique :db.unique/identity} 10 | :route/services {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many} 11 | :route/tags {:db/cardinality :db.cardinality/many} 12 | :service/trips {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/isComponent true} 13 | :trip/id {:db/unique :db.unique/identity} 14 | :service/id {:db/unique :db.unique/identity} 15 | :service/label {} 16 | :service/allocated-vessel {:db/valueType :db.type/ref :db/cardinality :db.cardinality/one} 17 | :vessel/imo {:db/unique :db.unique/identity} 18 | :vessel/name {}}) 19 | 20 | 21 | (def attrs (ch/find-attrs schema)) 22 | 23 | (deftest creates-refs-lookup 24 | (testing "creates new eids for unknown entity refs" 25 | (is (= (sut/create-refs-lookup {:from 1024 :to 2048} 26 | {[:route/number "100"] 1024 27 | [:vessel/imo "123"] 1025} 28 | [[:route/number "100"] 29 | [:vessel/imo "123"] 30 | [:service/id :s567]]) 31 | {[:route/number "100"] 1024 32 | [:vessel/imo "123"] 1025 33 | [:service/id :s567] 1026}))) 34 | 35 | (testing "uses db/id when given" 36 | (is (= (sut/create-refs-lookup {:from 1024 :to 2048} 37 | {[:route/number "100"] 1024 38 | [:vessel/imo "123"] 1025} 39 | [[:route/number "100"] 40 | [:vessel/imo "123"] 41 | [:db/id 99999]]) 42 | {[:route/number "100"] 1024 43 | [:vessel/imo "123"] 1025 44 | [:db/id 99999] 99999}))) 45 | 46 | (testing "creates only eids within given db-id-partition" 47 | (is (= (sut/create-refs-lookup {:from 1024 :to 2048} 48 | {[:route/number "100"] 1024 49 | [:db/id 99999] 99999} 50 | [[:route/number "100"] 51 | [:vessel/imo "123"] 52 | [:db/id 99999]]) 53 | {[:route/number "100"] 1024 54 | [:vessel/imo "123"] 1025 55 | [:db/id 99999] 99999}))) 56 | 57 | (testing "disallows setting db/id to a number within db-id-partition" 58 | (is (thrown? Exception 59 | (sut/create-refs-lookup {:from 1024 :to 2048} 60 | {[:route/number "100"] 1024 61 | [:vessel/imo "123"] 1025} 62 | [[:route/number "100"] 63 | [:vessel/imo "123"] 64 | [:db/id 2048]])))) 65 | 66 | (testing "fails when creating an internal eid outside the db-id-partition" 67 | (is (thrown? Exception 68 | (sut/create-refs-lookup {:from 1024 :to 2048} 69 | {[:route/number "100"] 2048} 70 | [[:route/number "100"] 71 | [:vessel/imo "123"]])))) 72 | 73 | ) 74 | 75 | (deftest flattens-entity-map 76 | (is (= (sut/flatten-entity-map attrs 77 | {[:route/number "100"] 1024 78 | [:vessel/imo "123"] 1025 79 | [:service/id :s567] 1026} 80 | {:route/number "100" 81 | :route/name "Stavanger-Tau"}) 82 | [[1024 :route/number "100"] 83 | [1024 :route/name "Stavanger-Tau"]])) 84 | 85 | ;; reverse entity ref, isn't component 86 | (is (= (sut/flatten-entity-map attrs 87 | {[:service/id :s567] 1026 88 | [:vessel/imo "123"] 1024} 89 | {:vessel/imo "123" 90 | :service/_allocated-vessel [{:service/id :s567}]}) 91 | [[1024 :vessel/imo "123"] 92 | [1026 :service/allocated-vessel 1024]])) 93 | 94 | ;; reverse entity ref, is component 95 | (is (= (sut/flatten-entity-map attrs 96 | {[:service/id :s567] 1026 97 | [:trip/id "foo"] 1025} 98 | {:trip/id "foo" 99 | :service/_trips {:service/id :s567}}) 100 | [[1025 :trip/id "foo"] 101 | [1026 :service/trips 1025]])) 102 | 103 | ;; using entity IDs for refs 104 | (is (= (sut/flatten-entity-map attrs 105 | {[:service/id :s567] 1024 106 | [:db/id 1025] 1025 107 | [:db/id 1026] 1026} 108 | {:service/id :s567 109 | :service/trips [1025 1026]}) 110 | [[1024 :service/id :s567] 111 | [1024 :service/trips 1025] 112 | [1024 :service/trips 1026]])) 113 | 114 | (is (= (sut/flatten-entity-map attrs 115 | {[:route/number "100"] 1024 116 | [:vessel/imo "123"] 1025 117 | [:service/id :s567] 1026} 118 | {:route/number "100" 119 | :route/services [{:service/id :s567 120 | :service/allocated-vessel {:vessel/imo "123"}}]}) 121 | [[1024 :route/number "100"] 122 | [1024 :route/services 1026]])) 123 | 124 | (is (= (sut/flatten-entity-map attrs 125 | {[:db/id 99999] 99999} 126 | {:db/id 99999 127 | :route/name "Stavanger-Tau"}) 128 | [[99999 :route/name "Stavanger-Tau"]])) 129 | 130 | (is (thrown? Exception ;; no nil vals 131 | (sut/flatten-entity-map attrs 132 | {[:route/number "100"] 1024 133 | [:vessel/imo "123"] 1025 134 | [:service/id :s567] 1026} 135 | {:route/number "100" 136 | :route/name nil})))) 137 | 138 | (deftest explodes 139 | (testing "no existing refs, simple case" 140 | (is (= (sut/explode {:schema schema :refs {}} 141 | [{:route/number "100"}]) 142 | {:refs {[:route/number "100"] 1024} 143 | :datoms #{[1024 :route/number "100"]}}))) 144 | 145 | (testing "cardinality many" 146 | (is (= (sut/explode {:schema schema :refs {}} 147 | [{:route/number "100" 148 | :route/tags #{:foo :bar :baz}}]) 149 | {:refs {[:route/number "100"] 1024} 150 | :datoms #{[1024 :route/number "100"] 151 | [1024 :route/tags :foo] 152 | [1024 :route/tags :bar] 153 | [1024 :route/tags :baz]}}))) 154 | 155 | (testing "no existing refs, interesting case" 156 | (is (= (sut/explode {:schema schema :refs {}} 157 | [{:route/number "100" 158 | :route/services [{:service/id :s567 159 | :service/allocated-vessel {:vessel/imo "123"}} 160 | {:service/id :s789}]}]) 161 | {:refs {[:route/number "100"] 1024 162 | [:service/id :s567] 1025 163 | [:service/id :s789] 1026 164 | [:vessel/imo "123"] 1027} 165 | :datoms #{[1024 :route/number "100"] 166 | [1024 :route/services 1025] 167 | [1024 :route/services 1026] 168 | [1025 :service/id :s567] 169 | [1025 :service/allocated-vessel 1027] 170 | [1026 :service/id :s789] 171 | [1027 :vessel/imo "123"]}}))) 172 | 173 | (testing "with db/id" 174 | (is (= (sut/explode {:schema schema :refs {}} 175 | [{:db/id 99999 176 | :route/name "Stavanger-Tau"}]) 177 | {:refs {[:db/id 99999] 99999} 178 | :datoms #{[99999 :route/name "Stavanger-Tau"]}}))) 179 | 180 | (testing "reverse refs" 181 | (is (= {:refs {[:route/number "100"] 1027 182 | [:service/id :s567] 1026 183 | [:service/id :s789] 1028 184 | [:trip/id "foo"] 1025 185 | [:trip/id "bar"] 1029 186 | [:vessel/imo "123"] 1024} 187 | :datoms #{[1024 :vessel/imo "123"] 188 | [1025 :trip/id "foo"] 189 | [1029 :trip/id "bar"] 190 | [1026 :service/allocated-vessel 1024] 191 | [1026 :service/id :s567] 192 | [1027 :route/number "100"] 193 | [1027 :route/services 1026] 194 | [1027 :route/services 1028] 195 | [1028 :service/id :s789] 196 | [1028 :service/trips 1025] 197 | [1028 :service/trips 1029]}} 198 | (sut/explode {:schema schema :refs {}} 199 | [{:vessel/imo "123" 200 | :service/_allocated-vessel [{:service/id :s567 201 | :route/_services #{{:route/number "100"}}}]} 202 | {:trip/id "foo" 203 | :service/_trips {:service/id :s789 204 | :route/_services #{{:route/number "100"}} 205 | :service/trips #{{:trip/id "bar"}}}}])))) 206 | 207 | (testing "existing refs" 208 | (is (= (sut/explode {:schema schema :refs {[:route/number "100"] 2024 209 | [:service/id :s567] 2025}} 210 | [{:route/number "100" 211 | :route/services [{:service/id :s567 212 | :service/allocated-vessel {:vessel/imo "123"}}]}]) 213 | {:refs {[:route/number "100"] 2024 214 | [:service/id :s567] 2025 215 | [:vessel/imo "123"] 2026} 216 | :datoms #{[2024 :route/number "100"] 217 | [2024 :route/services 2025] 218 | [2025 :service/id :s567] 219 | [2025 :service/allocated-vessel 2026] 220 | [2026 :vessel/imo "123"]}}))) 221 | 222 | (testing "conflicting values for entity attr" 223 | (try 224 | (sut/explode {:schema schema :refs {}} 225 | [{:route/number "100" :route/name "Stavanger-Taua"} 226 | {:route/number "100" :route/name "Stavanger-Tau"}]) 227 | (is (= :should-throw :didnt)) 228 | (catch Exception e 229 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 230 | (is (= {:entity-ref [:route/number "100"] 231 | :attr :route/name 232 | :conflicting-values #{"Stavanger-Taua" "Stavanger-Tau"}} 233 | (ex-data e)))))) 234 | 235 | (testing "no attrs asserted for entity" 236 | (is (thrown? Exception 237 | (sut/explode {:schema schema :refs {}} 238 | [{:db/id 999999}]))) 239 | 240 | (is (thrown? Exception 241 | (sut/explode {:schema schema :refs {}} 242 | [{:db/id 999999 :route/services [{:db/id 888888}]}]))) 243 | 244 | ;; adding refs to an existing entity is OK 245 | (is (= {:refs {[:db/id 999999] 999999 246 | [:db/id 888888] 888888} 247 | :datoms #{[999999 :route/services 888888]}} 248 | (sut/explode {:schema schema :refs {[:db/id 888888] 888888}} 249 | [{:db/id 999999 :route/services [{:db/id 888888}]}]))) 250 | 251 | (is (= (sut/explode {:schema schema :refs {}} 252 | [{:db/id 999999 :route/services [{:db/id 888888 :service/name "Tjeneste 1"}]}]) 253 | (sut/explode {:schema schema :refs {}} 254 | [{:db/id 999999 :route/services [{:db/id 888888}]} 255 | {:db/id 888888 :service/name "Tjeneste 1"}]))))) 256 | 257 | (deftest diffs 258 | (is (= (sut/diff #{} 259 | #{[2025 :service/id :s567] 260 | [2025 :service/allocated-vessel 2026] 261 | [2026 :vessel/imo "123"]}) 262 | [[:db/add 2025 :service/id :s567] 263 | [:db/add 2025 :service/allocated-vessel 2026] 264 | [:db/add 2026 :vessel/imo "123"]])) 265 | 266 | (is (= (sut/diff #{[2025 :service/id :s567] 267 | [2025 :service/allocated-vessel 2026] 268 | [2026 :vessel/imo "123"]} 269 | #{}) 270 | [[:db/retract 2025 :service/id :s567] 271 | [:db/retract 2025 :service/allocated-vessel 2026] 272 | [:db/retract 2026 :vessel/imo "123"]])) 273 | 274 | (is (= (sut/diff #{[2024 :service/id :s345] [2025 :service/id :s567] [2025 :service/allocated-vessel 2026] [2026 :vessel/imo "123"]} 275 | #{[2024 :service/id :s345] [2025 :service/id :s567] [2024 :service/allocated-vessel 2026] [2026 :vessel/imo "123"]}) 276 | [[:db/retract 2025 :service/allocated-vessel 2026] 277 | [:db/add 2024 :service/allocated-vessel 2026]]))) 278 | 279 | (deftest with-keeps-track-of-different-sources 280 | (let [db-at-first (sut/empty-db schema) 281 | tx-sporadic-1 [{:route/number "100" 282 | :route/services [{:service/id :s567 283 | :service/allocated-vessel {:vessel/imo "123"}}]}] 284 | refs {[:route/number "100"] 1024 285 | [:service/id :s567] 1025 286 | [:vessel/imo "123"] 1026} 287 | 288 | sporadic-1-datoms #{[1024 :route/number "100"] 289 | [1024 :route/services 1025] 290 | [1025 :service/id :s567] 291 | [1025 :service/allocated-vessel 1026] 292 | [1026 :vessel/imo "123"]} 293 | 294 | tx-data-sporadic-1 #{[:db/add 1024 :route/number "100"] 295 | [:db/add 1024 :route/services 1025] 296 | [:db/add 1025 :service/id :s567] 297 | [:db/add 1025 :service/allocated-vessel 1026] 298 | [:db/add 1026 :vessel/imo "123"]} 299 | 300 | db-after-sporadic-1 {:schema schema 301 | :refs refs 302 | :source-datoms {:sporadic sporadic-1-datoms}} 303 | 304 | tx-frequent-1 [{:vessel/imo "123" :vessel/name "Jekyll"}] 305 | 306 | tx-data-frequent-1 #{[:db/add 1026 :vessel/name "Jekyll"]} 307 | 308 | db-after-frequent-1 {:schema schema 309 | :refs refs 310 | :source-datoms {:sporadic sporadic-1-datoms 311 | :frequent #{[1026 :vessel/imo "123"] 312 | [1026 :vessel/name "Jekyll"]}}} 313 | 314 | tx-frequent-2 [] 315 | 316 | tx-data-frequent-2 #{[:db/retract 1026 :vessel/name "Jekyll"]} 317 | 318 | db-after-frequent-2 {:schema schema 319 | :refs refs 320 | :source-datoms {:sporadic sporadic-1-datoms 321 | :frequent #{}}} 322 | 323 | tx-sporadic-2 [{:route/number "100" 324 | :route/services [{:service/id :s567}]}] 325 | 326 | tx-data-sporadic-2 #{[:db/retract 1025 :service/allocated-vessel 1026] 327 | [:db/retract 1026 :vessel/imo "123"]} 328 | 329 | db-after-sporadic-2 {:schema schema 330 | :refs refs 331 | :source-datoms {:sporadic #{[1024 :route/number "100"] 332 | [1024 :route/services 1025] 333 | [1025 :service/id :s567]} 334 | :frequent #{}}}] 335 | (is (= (-> (sut/with db-at-first :sporadic tx-sporadic-1) 336 | (update :tx-data set)) 337 | {:tx-data tx-data-sporadic-1 338 | :db-before db-at-first 339 | :db-after db-after-sporadic-1})) 340 | 341 | (is (= (-> (sut/with db-after-sporadic-1 :frequent tx-frequent-1) 342 | (update :tx-data set)) 343 | {:tx-data tx-data-frequent-1 344 | :db-before db-after-sporadic-1 345 | :db-after db-after-frequent-1})) 346 | 347 | (is (= (-> (sut/with db-after-frequent-1 :frequent tx-frequent-2) 348 | (update :tx-data set)) 349 | {:tx-data tx-data-frequent-2 350 | :db-before db-after-frequent-1 351 | :db-after db-after-frequent-2})) 352 | 353 | (is (= (-> (sut/with db-after-frequent-2 :sporadic tx-sporadic-2) 354 | (update :tx-data set)) 355 | {:tx-data tx-data-sporadic-2 356 | :db-before db-after-frequent-2 357 | :db-after db-after-sporadic-2})))) 358 | 359 | (deftest with-complains-about-conflicting-information-from-different-sources 360 | (testing "throws when sources give different values for same attribute" 361 | (let [db-at-first (sut/empty-db schema) 362 | tx-source-1 [{:service/id :s567 :service/label "I"}] 363 | tx-source-2 [{:service/id :s567 :service/label "II"}]] 364 | (try (-> db-at-first 365 | (sut/with :source-1 tx-source-1) 366 | :db-after 367 | (sut/with :source-2 tx-source-2)) 368 | (is (= :should-throw :didnt)) 369 | (catch Exception e 370 | (is (= "Conflicting values asserted between sources" (.getMessage e))) 371 | (is (= {:source-1 [[:service/id :s567] :service/label "I"] 372 | :source-2 [[:service/id :s567] :service/label "II"]} 373 | (ex-data e))))))) 374 | 375 | (testing "includes entity ref for conflicting ref values" 376 | (let [db-at-first (sut/empty-db schema) 377 | tx-source-1 [{:service/id :s567 :service/allocated-vessel {:vessel/imo "123"}}] 378 | tx-source-2 [{:service/id :s567 :service/allocated-vessel {:vessel/imo "456"}}]] 379 | (try (-> db-at-first 380 | (sut/with :source-1 tx-source-1) 381 | :db-after 382 | (sut/with :source-2 tx-source-2)) 383 | (is (= :should-throw :didnt)) 384 | (catch Exception e 385 | (is (= "Conflicting values asserted between sources" (.getMessage e))) 386 | (is (= {:source-1 [[:service/id :s567] :service/allocated-vessel [:vessel/imo "123"]] 387 | :source-2 [[:service/id :s567] :service/allocated-vessel [:vessel/imo "456"]]} 388 | (ex-data e))))))) 389 | 390 | (testing "does not throw when new entities are asserted for a cardinality many attr" 391 | (let [db-at-first (sut/empty-db schema) 392 | tx-source-1 [{:service/id :s567 :service/trips #{{:trip/id "bar"}}}] 393 | tx-source-2 [{:service/id :s567 :service/trips #{{:trip/id "foo"}}}]] 394 | (is (-> db-at-first 395 | (sut/with :source-1 tx-source-1) 396 | :db-after 397 | (sut/with :source-2 tx-source-2)))))) 398 | 399 | (deftest with-sources 400 | (testing "throws when sources give different values for same attribute" 401 | (let [db-at-first (sut/empty-db schema) 402 | tx-source-1-beg [{:service/id :s567 :service/label "I"}] 403 | tx-source-2-beg [{:service/id :s567 :service/label "I"}] 404 | tx-source-1-end [{:service/id :s567 :service/label "II"}] 405 | tx-source-2-end [{:service/id :s567 :service/label "II"}]] 406 | (is (-> db-at-first 407 | (sut/with :source-1 tx-source-1-beg) 408 | :db-after 409 | (sut/with :source-2 tx-source-2-beg) 410 | :db-after 411 | (sut/with-sources {:source-1 tx-source-1-end 412 | :source-2 tx-source-2-end})))))) 413 | 414 | (deftest with-supports-partial-updates 415 | (let [db-at-first (sut/empty-db schema) 416 | tx-1 [{:route/number "100" 417 | :route/services [{:service/id :s567 418 | :service/allocated-vessel {:vessel/imo "123"}}]}] 419 | refs {[:route/number "100"] 1024 420 | [:service/id :s567] 1025 421 | [:vessel/imo "123"] 1026} 422 | 423 | tx-1-datoms #{[1024 :route/number "100"] 424 | [1024 :route/services 1025] 425 | [1025 :service/id :s567] 426 | [1025 :service/allocated-vessel 1026] 427 | [1026 :vessel/imo "123"]} 428 | 429 | tx-1-data #{[:db/add 1024 :route/number "100"] 430 | [:db/add 1024 :route/services 1025] 431 | [:db/add 1025 :service/id :s567] 432 | [:db/add 1025 :service/allocated-vessel 1026] 433 | [:db/add 1026 :vessel/imo "123"]} 434 | 435 | db-after-tx-1 {:schema schema 436 | :refs refs 437 | :source-datoms {:source tx-1-datoms}} 438 | 439 | tx-2 (with-meta 440 | [{:route/number "100" 441 | :route/name "Stavanger-Tau"}] 442 | {:partial-update? true}) 443 | 444 | tx-2-datoms #{[1024 :route/number "100"] 445 | [1024 :route/name "Stavanger-Tau"] 446 | [1024 :route/services 1025] 447 | [1025 :service/id :s567] 448 | [1025 :service/allocated-vessel 1026] 449 | [1026 :vessel/imo "123"]} 450 | 451 | tx-2-data #{[:db/add 1024 :route/name "Stavanger-Tau"]} 452 | 453 | db-after-tx-2 {:schema schema 454 | :refs refs 455 | :source-datoms {:source tx-2-datoms}}] 456 | 457 | (is (= (-> (sut/with db-at-first :source tx-1) 458 | (update :tx-data set)) 459 | {:tx-data tx-1-data 460 | :db-before db-at-first 461 | :db-after db-after-tx-1})) 462 | 463 | (is (= (-> (sut/with db-after-tx-1 :source tx-2) 464 | (update :tx-data set)) 465 | {:tx-data tx-2-data 466 | :db-before db-after-tx-1 467 | :db-after db-after-tx-2})))) 468 | -------------------------------------------------------------------------------- /test/datoms_differ/api_test.cljc: -------------------------------------------------------------------------------- 1 | (ns datoms-differ.api-test 2 | (:require [datoms-differ.api :as sut] 3 | [datoms-differ.datom :as d] 4 | [clojure.set :as set] 5 | [clojure.test :refer [deftest is testing]]) 6 | (:import (java.lang Exception))) 7 | 8 | (def schema 9 | {::sut/db-id-partition {:from 1024 :to 2048} 10 | :route/name {} 11 | :route/number {:db/unique :db.unique/identity} 12 | :route/services {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many} 13 | :route/tags {:db/cardinality :db.cardinality/many} 14 | :route/holidays {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many} 15 | :service/trips {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/isComponent true} 16 | :trip/id {:db/unique :db.unique/identity} 17 | :service/id {:db/unique :db.unique/identity} 18 | :service/label {} 19 | :service/allocated-vessel {:db/valueType :db.type/ref :db/cardinality :db.cardinality/one} 20 | :vessel/imo {:db/unique :db.unique/identity} 21 | :vessel/name {} 22 | :holiday/bus-key {:db/unique :db.unique/identity 23 | :db/tupleAttrs [:holiday/pattern :holiday/weekday]} 24 | :holiday/pattern {} 25 | :holiday/weekday {} 26 | :my-tuple/id {:db/unique :db.unique/identity} 27 | :my-tuple/a {} 28 | :my-tuple/b {} 29 | :my-tuple/tup {:db/tupleAttrs [:my-tuple/a :my-tuple/b]}}) 30 | 31 | (deftest create-conn 32 | (is (= @(sut/create-conn schema) 33 | {:schema schema 34 | :refs {} 35 | :eavs #{}}))) 36 | 37 | (defn assert-report [report {:keys [tx-data refs eavs refs-added eavs-added]}] 38 | (when tx-data (is (= (:tx-data report) tx-data) "Difference in tx-data")) 39 | (when refs (is (= (-> report :db-after :refs) refs) "Difference in refs")) 40 | (when eavs (is (= (-> report :db-after :eavs) eavs) "Difference in eavs")) 41 | (when refs-added 42 | (is (= (-> report :db-after :refs) 43 | (merge (-> report :db-after :refs) refs-added)) 44 | "Difference in refs (added)")) 45 | (when eavs-added 46 | (is (= (-> report :db-after :eavs) 47 | (set/union (-> report :db-after :eavs) eavs-added)) 48 | "Difference in eavs (added)"))) 49 | 50 | (defn db [] 51 | @(sut/create-conn schema)) 52 | 53 | (deftest with 54 | (testing "EMPTY DB, ONE SOURCE" 55 | 56 | (testing "empty db no entities returns same" 57 | (let [empty-db (db)] 58 | (is (= (sut/with empty-db :prepare []) 59 | {:tx-data [] 60 | :db-before empty-db 61 | :db-after empty-db})))) 62 | 63 | (testing "empty db one entity" 64 | (assert-report 65 | (sut/with (db) :prepare-routes [{:route/number "800" 66 | :route/name "Røvær"}]) 67 | {:tx-data [[:db/add 1024 :route/name "Røvær"] 68 | [:db/add 1024 :route/number "800"]] 69 | :eavs #{(d/datom 1024 :route/number "800" :prepare-routes) 70 | (d/datom 1024 :route/name "Røvær" :prepare-routes)} 71 | :refs {[:route/number "800"] 1024}})) 72 | 73 | (testing "non-keyword source" 74 | (assert-report 75 | (sut/with (db) [:prepare-routes 1] [{:route/number "800"}]) 76 | {:tx-data [[:db/add 1024 :route/number "800"]] 77 | :eavs #{(d/datom 1024 :route/number "800" [:prepare-routes 1])} 78 | :refs {[:route/number "800"] 1024}})) 79 | 80 | (testing "empty db one entity and cardinality many" 81 | (assert-report 82 | (sut/with (db) :prepare-routes [{:route/number "800" 83 | :route/tags #{:tag-1 :tag-2}}]) 84 | 85 | {:tx-data [[:db/add 1024 :route/number "800"] 86 | [:db/add 1024 :route/tags :tag-1] 87 | [:db/add 1024 :route/tags :tag-2]] 88 | 89 | :eavs #{(d/datom 1024 :route/number "800" :prepare-routes) 90 | (d/datom 1024 :route/tags :tag-1 :prepare-routes) 91 | (d/datom 1024 :route/tags :tag-2 :prepare-routes)} 92 | :refs {[:route/number "800"] 1024}})) 93 | 94 | (testing "empty db entity with tuple attributes" 95 | (assert-report 96 | (sut/with (db) :prepare [{:holiday/pattern :christmas-day :holiday/weekday :monday}]) 97 | 98 | {:tx-data [[:db/add 1024 :holiday/bus-key [:christmas-day :monday]] 99 | [:db/add 1024 :holiday/pattern :christmas-day] 100 | [:db/add 1024 :holiday/weekday :monday]] 101 | 102 | :eavs #{(d/datom 1024 :holiday/bus-key [:christmas-day :monday] :prepare) 103 | (d/datom 1024 :holiday/pattern :christmas-day :prepare) 104 | (d/datom 1024 :holiday/weekday :monday :prepare)} 105 | :refs {[:holiday/bus-key [:christmas-day :monday]] 1024}})) 106 | 107 | (testing "empty db one entity with nested entities" 108 | (assert-report 109 | (sut/with (db) 110 | :prepare-routes [{:route/number "800" 111 | :route/name "Røvær" 112 | :route/services [{:service/id :service-1 113 | :service/label "service-label"}]}]) 114 | 115 | {:tx-data [[:db/add 1024 :route/name "Røvær"] 116 | [:db/add 1024 :route/number "800"] 117 | [:db/add 1024 :route/services 1025] 118 | [:db/add 1025 :service/id :service-1] 119 | [:db/add 1025 :service/label "service-label"]] 120 | :eavs #{(d/datom 1024 :route/number "800" :prepare-routes) 121 | (d/datom 1024 :route/name "Røvær" :prepare-routes) 122 | (d/datom 1024 :route/services 1025 :prepare-routes) 123 | (d/datom 1025 :service/id :service-1 :prepare-routes) 124 | (d/datom 1025 :service/label "service-label" :prepare-routes)} 125 | :refs {[:route/number "800"] 1024 126 | [:service/id :service-1] 1025}})) 127 | 128 | (testing "empty db entity with nested entitites with tuple attributes" 129 | (assert-report 130 | (sut/with (db) :prepare [{:route/number "800" 131 | :route/holidays [{:holiday/pattern :christmas-day :holiday/weekday :monday}]}]) 132 | 133 | {:tx-data [[:db/add 1024 :route/holidays 1025] 134 | [:db/add 1024 :route/number "800"] 135 | [:db/add 1025 :holiday/bus-key [:christmas-day :monday]] 136 | [:db/add 1025 :holiday/pattern :christmas-day] 137 | [:db/add 1025 :holiday/weekday :monday]] 138 | 139 | :eavs #{(d/datom 1024 :route/holidays 1025 :prepare) 140 | (d/datom 1024 :route/number "800" :prepare) 141 | (d/datom 1025 :holiday/bus-key [:christmas-day :monday] :prepare) 142 | (d/datom 1025 :holiday/pattern :christmas-day :prepare) 143 | (d/datom 1025 :holiday/weekday :monday :prepare)} 144 | :refs {[:route/number "800"] 1024 145 | [:holiday/bus-key [:christmas-day :monday]] 1025}})) 146 | 147 | (testing "empty db - reverse entity-ref, isn't component" 148 | (assert-report 149 | (sut/with 150 | (db) 151 | :prepare-routes [{:vessel/imo "123" 152 | :service/_allocated-vessel [{:service/id :s567}]}]) 153 | 154 | {:tx-data [[:db/add 1024 :vessel/imo "123"] 155 | [:db/add 1025 :service/allocated-vessel 1024] 156 | [:db/add 1025 :service/id :s567]] 157 | :eavs #{(d/datom 1024 :vessel/imo "123" :prepare-routes) 158 | (d/datom 1025 :service/id :s567 :prepare-routes) 159 | (d/datom 1025 :service/allocated-vessel 1024 :prepare-routes)} 160 | :refs {[:vessel/imo "123"] 1024 161 | [:service/id :s567] 1025}})) 162 | 163 | (testing "empty db - reverse entity-ref, is component" 164 | (assert-report 165 | (sut/with 166 | (db) 167 | :prepare-routes [{:trip/id "foo" 168 | :service/_trips {:service/id :s567}}]) 169 | 170 | {:tx-data [[:db/add 1024 :trip/id "foo"] 171 | [:db/add 1025 :service/id :s567] 172 | [:db/add 1025 :service/trips 1024]] 173 | :eavs #{(d/datom 1024 :trip/id "foo" :prepare-routes) 174 | (d/datom 1025 :service/id :s567 :prepare-routes) 175 | (d/datom 1025 :service/trips 1024 :prepare-routes)} 176 | :refs {[:trip/id "foo"] 1024 177 | [:service/id :s567] 1025}})) 178 | 179 | (testing "empty db - reverse entity-ref - tuple style" 180 | (assert-report 181 | (sut/with (db) :prepare [{:holiday/pattern :christmas-day 182 | :holiday/weekday :monday 183 | :route/_holidays [{:route/number "800"}]}]) 184 | 185 | {:tx-data [[:db/add 1024 :holiday/bus-key [:christmas-day :monday]] 186 | [:db/add 1024 :holiday/pattern :christmas-day] 187 | [:db/add 1024 :holiday/weekday :monday] 188 | [:db/add 1025 :route/holidays 1024] 189 | [:db/add 1025 :route/number "800"]] 190 | 191 | :eavs #{(d/datom 1024 :holiday/bus-key [:christmas-day :monday] :prepare) 192 | (d/datom 1024 :holiday/pattern :christmas-day :prepare) 193 | (d/datom 1024 :holiday/weekday :monday :prepare) 194 | (d/datom 1025 :route/holidays 1024 :prepare) 195 | (d/datom 1025 :route/number "800" :prepare)} 196 | :refs {[:holiday/bus-key [:christmas-day :monday]] 1024 197 | [:route/number "800"] 1025}})) 198 | 199 | (testing "empty-db - reverse refs multiple levels" 200 | (assert-report 201 | (sut/with 202 | (db) 203 | :prep [{:vessel/imo "123" 204 | :service/_allocated-vessel [{:service/id :s567 205 | :route/_services #{{:route/number "100"}}}]} 206 | {:trip/id "foo" 207 | :service/_trips {:service/id :s789 208 | :route/_services #{{:route/number "100"}} 209 | :service/trips #{{:trip/id "bar"}}}}]) 210 | {:tx-data [[:db/add 1024 :vessel/imo "123"] 211 | [:db/add 1025 :trip/id "foo"] 212 | [:db/add 1026 :service/allocated-vessel 1024] 213 | [:db/add 1026 :service/id :s567] 214 | [:db/add 1027 :route/number "100"] 215 | [:db/add 1027 :route/services 1026] 216 | [:db/add 1027 :route/services 1028] 217 | [:db/add 1028 :service/id :s789] 218 | [:db/add 1028 :service/trips 1025] 219 | [:db/add 1028 :service/trips 1029] 220 | [:db/add 1029 :trip/id "bar"]] 221 | :eavs #{(d/datom 1024 :vessel/imo "123" :prep) 222 | (d/datom 1025 :trip/id "foo" :prep) 223 | (d/datom 1029 :trip/id "bar" :prep) 224 | (d/datom 1026 :service/allocated-vessel 1024 :prep) 225 | (d/datom 1026 :service/id :s567 :prep) 226 | (d/datom 1027 :route/number "100" :prep) 227 | (d/datom 1027 :route/services 1026 :prep) 228 | (d/datom 1027 :route/services 1028 :prep) 229 | (d/datom 1028 :service/id :s789 :prep) 230 | (d/datom 1028 :service/trips 1025 :prep) 231 | (d/datom 1028 :service/trips 1029 :prep)} 232 | :refs {[:route/number "100"] 1027 233 | [:service/id :s567] 1026 234 | [:service/id :s789] 1028 235 | [:trip/id "foo"] 1025 236 | [:trip/id "bar"] 1029 237 | [:vessel/imo "123"] 1024}})) 238 | 239 | (testing "empty db - nil val for attr throws" 240 | (is (thrown-with-msg? Exception #"Attributes cannot be nil" 241 | (sut/with (db) :prepare-routes [{:route/number "100" 242 | :route/name nil}])))) 243 | 244 | (testing "Running out of eids for db-id-partition throws" 245 | (let [db (sut/create-conn (assoc schema ::sut/db-id-partition {:from 1 :to 2}))] 246 | (is 247 | (thrown-with-msg? Exception #"Generated internal eid falls outside internal db-id-partition," 248 | (sut/with @db :prep [{:vessel/imo "123"} 249 | {:vessel/imo "234"} 250 | {:vessel/imo "345"}]))))) 251 | 252 | (testing "empty db - entity without identity attributes throws" 253 | (is (thrown? Exception 254 | (sut/with (db) :prepare-routes [{}]))) 255 | (is (thrown? Exception 256 | (sut/with (db) :prepare-routes [{:route/name "Dufus"}])))) 257 | 258 | (testing "conflicting values for entity attr throws" 259 | (try 260 | (sut/with (db) :prepare-routes [{:route/number "100" :route/name "Stavanger-Taua"} 261 | {:route/number "100" :route/name "Stavanger-Tau"}]) 262 | (is (= :should-throw :didnt)) 263 | (catch Exception e 264 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 265 | (is (= {:entity-ref [:route/number "100"] 266 | :attr :route/name 267 | :conflict {:prepare-routes #{"Stavanger-Taua" "Stavanger-Tau"}}} 268 | (ex-data e)))))) 269 | 270 | (testing "tuple unique identity conflicts on two entities with same tuple throws" 271 | (try 272 | (sut/with (db) :prepare [{:holiday/pattern :christmas-day :holiday/weekday :monday :holiday/extra :foo} 273 | {:holiday/pattern :christmas-day :holiday/weekday :monday :holiday/extra :bar}]) 274 | 275 | (is (= :should-throw :didnt)) 276 | (catch Exception e 277 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 278 | (is (= {:entity-ref [:holiday/bus-key [:christmas-day :monday]] 279 | :attr :holiday/extra 280 | :conflict {:prepare #{:foo :bar}}} 281 | (ex-data e)))) 282 | ))) 283 | 284 | (testing "PREVIOUS DB, ONE SOURCE" 285 | (testing "Same entitie(s) gives no diff" 286 | (let [{:keys [db-after]} (sut/with (db) :prepare-routes [{:route/number "800" :route/name "Røvær"}])] 287 | (assert-report 288 | (sut/with db-after :prepare-routes [{:route/number "800" :route/name "Røvær"}]) 289 | {:tx-data [] 290 | :refs (:refs db-after) 291 | :eavs (:eavs db-after)}))) 292 | 293 | (testing "Added attr to entity gives txes and new datoms" 294 | (let [{:keys [db-after]} (sut/with (db) :prepare-routes [{:route/number "800" 295 | :route/name "Røvær"}])] 296 | (assert-report 297 | (sut/with db-after :prepare-routes [{:route/number "800" 298 | :route/name "Røvær" 299 | :route/tags #{:tag-1 :tag-2}}]) 300 | 301 | {:tx-data [[:db/add 1024 :route/tags :tag-1] 302 | [:db/add 1024 :route/tags :tag-2]] 303 | :eavs-added #{(d/datom 1024 :route/tags :tag-1 :prepare-routes) 304 | (d/datom 1024 :route/tags :tag-2 :prepare-routes)} 305 | :refs-added {}}))) 306 | 307 | (testing "Added new entity gives txes, new datoms and new refs" 308 | (let [{:keys [db-after]} (sut/with (db) :prepare-routes [{:route/number "800" 309 | :route/name "Røvær"}])] 310 | (assert-report 311 | (sut/with db-after :prepare-routes [{:route/number "800" 312 | :route/name "Røvær"} 313 | {:route/number "700" 314 | :route/name "Døvær"}]) 315 | 316 | {:tx-data [[:db/add 1025 :route/name "Døvær"] 317 | [:db/add 1025 :route/number "700"]] 318 | :eavs-added #{(d/datom 1025 :route/number "700" :prepare-routes) 319 | (d/datom 1025 :route/name "Døvær" :prepare-routes)} 320 | :refs-added {[:route/number "700"] 1025}}))) 321 | 322 | (testing "Added new entity and remove one gives add/remove txes, new datoms and new refs" 323 | (let [{:keys [db-after]} (sut/with (db) :prepare-routes [{:route/number "800" 324 | :route/name "Røvær"}])] 325 | (assert-report 326 | (sut/with db-after :prepare-routes [{:route/number "700" 327 | :route/name "Døvær"}]) 328 | 329 | {:tx-data [[:db/retract 1024 :route/name "Røvær"] 330 | [:db/retract 1024 :route/number "800"] 331 | [:db/add 1025 :route/name "Døvær"] 332 | [:db/add 1025 :route/number "700"]] 333 | :eavs #{(d/datom 1025 :route/number "700" :prepare-routes) 334 | (d/datom 1025 :route/name "Døvær" :prepare-routes)} 335 | :refs {[:route/number "700"] 1025}}))) 336 | 337 | (testing "Added new entity and remove one gives add/remove txes, new datoms and new refs" 338 | (let [{:keys [db-after]} (sut/with (db) :prepare-routes [{:route/number "800" 339 | :route/name "Røvær"}])] 340 | (assert-report 341 | (sut/with db-after :prepare-routes [{:route/number "700" 342 | :route/name "Døvær"}]) 343 | 344 | {:tx-data [[:db/retract 1024 :route/name "Røvær"] 345 | [:db/retract 1024 :route/number "800"] 346 | [:db/add 1025 :route/name "Døvær"] 347 | [:db/add 1025 :route/number "700"]] 348 | :eavs #{(d/datom 1025 :route/number "700" :prepare-routes) 349 | (d/datom 1025 :route/name "Døvær" :prepare-routes)} 350 | :refs {[:route/number "700"] 1025}}))) 351 | 352 | (testing "Remove all attributes in generated tuple also retracts tupleAttr" 353 | (let [entities [{:my-tuple/id :one 354 | :my-tuple/a :a 355 | :my-tuple/b :b}] 356 | {:keys [db-after]} (sut/with (db) :prepare entities)] 357 | (assert-report 358 | (sut/with db-after :prepare [{:my-tuple/id :one}]) 359 | 360 | {:tx-data [[:db/retract 1024 :my-tuple/a :a] 361 | [:db/retract 1024 :my-tuple/b :b] 362 | [:db/retract 1024 :my-tuple/tup [:a :b]]] 363 | :refs {[:my-tuple/id :one] 1024}}))) 364 | 365 | (testing "Update of attributes used in tupleAttr also updates the tupleAttr " 366 | (let [entities [{:my-tuple/id :one 367 | :my-tuple/a :a 368 | :my-tuple/b :b}] 369 | {:keys [db-after]} (sut/with (db) :prepare entities)] 370 | (assert-report 371 | (sut/with db-after :prepare [{:my-tuple/id :one 372 | :my-tuple/a :a}]) 373 | 374 | {:tx-data [[:db/retract 1024 :my-tuple/b :b] 375 | [:db/retract 1024 :my-tuple/tup [:a :b]] 376 | [:db/add 1024 :my-tuple/tup [:a nil]]] 377 | :refs {[:my-tuple/id :one] 1024}}))) 378 | 379 | (testing "Modify value to conflict for optimized check throws" 380 | (let [entities (concat [{:vessel/imo "123" :vessel/name "Fyken"}] 381 | (for [i (range 100)] 382 | {:vessel/imo (str i) :vessel/name (str i)}))] 383 | (try (-> (sut/with (db) :source entities) 384 | :db-after 385 | (sut/with :source (concat entities [{:vessel/imo "123" :vessel/name "Syken"}]))) 386 | (is (= :should-throw :didnt)) 387 | (catch Exception e 388 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 389 | (is (= {:attr :vessel/name 390 | :entity-ref [:vessel/imo "123"] 391 | :conflict {:source #{"Fyken" "Syken"}}} 392 | (ex-data e))))))))) 393 | 394 | (deftest with-sources 395 | (testing "EMPTY DB, MULTIPLE SOURCES" 396 | (testing "empty db two sources " 397 | (assert-report 398 | (sut/with-sources 399 | (db) 400 | {:prepare-routes [{:vessel/imo "123" 401 | :vessel/name "Fyken"}] 402 | :prepare-services [{:service/id :s123 403 | :service/allocated-vessel {:vessel/imo "123"}}]}) 404 | 405 | {:tx-data [[:db/add 1024 :vessel/imo "123"] 406 | [:db/add 1024 :vessel/name "Fyken"] 407 | [:db/add 1025 :service/allocated-vessel 1024] 408 | [:db/add 1025 :service/id :s123]] 409 | :eavs #{(d/datom 1024 :vessel/imo "123" :prepare-routes) 410 | (d/datom 1024 :vessel/name "Fyken" :prepare-routes) 411 | (d/datom 1025 :service/id :s123 :prepare-services) 412 | (d/datom 1025 :service/allocated-vessel 1024 :prepare-services) 413 | (d/datom 1024 :vessel/imo "123" :prepare-services)} 414 | :refs {[:vessel/imo "123"] 1024 415 | [:service/id :s123] 1025}})) 416 | 417 | (testing "empty db conflicting sources throws" 418 | (try (sut/with-sources 419 | (db) 420 | {:prepare-vessel [{:vessel/imo "123" :vessel/name "Fyken"}] 421 | :prepare-vessel-2 [{:vessel/imo "123" :vessel/name "Syken"}]}) 422 | (is (= :should-throw :didnt)) 423 | (catch Exception e 424 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 425 | (is (= {:attr :vessel/name 426 | :entity-ref [:vessel/imo "123"] 427 | :conflict {:prepare-vessel #{"Fyken"} 428 | :prepare-vessel-2 #{"Syken"}}} 429 | (ex-data e)))))) 430 | 431 | (testing "empty db conflicting values throws and includes entity ref for refs" 432 | (try (sut/with-sources 433 | (db) 434 | {:source-1 [{:service/id :s567 :service/allocated-vessel {:vessel/imo "123"}}] 435 | :source-2 [{:service/id :s567 :service/allocated-vessel {:vessel/imo "456"}}]}) 436 | (is (= :should-throw :didnt)) 437 | (catch Exception e 438 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 439 | (is (= {:attr :service/allocated-vessel 440 | :entity-ref [:service/id :s567] 441 | :conflict {:source-1 #{[:vessel/imo "123"]} 442 | :source-2 #{[:vessel/imo "456"]}}} 443 | (ex-data e)))))) 444 | 445 | (testing "does not throw when new entities are asserted for a cardinality many attr" 446 | (let [db-at-first (db) 447 | tx-source-1 [{:service/id :s567 :service/trips #{{:trip/id "bar"}}}] 448 | tx-source-2 [{:service/id :s567 :service/trips #{{:trip/id "foo"}}}]] 449 | (is (= (-> db-at-first 450 | (sut/with :source-1 tx-source-1) 451 | :db-after 452 | (sut/with :source-2 tx-source-2) 453 | :db-after 454 | :eavs) 455 | #{(d/datom 1024 :service/id :s567 :source-1) 456 | (d/datom 1024 :service/id :s567 :source-2) 457 | (d/datom 1024 :service/trips 1025 :source-1) 458 | (d/datom 1024 :service/trips 1026 :source-2) 459 | (d/datom 1025 :trip/id "bar" :source-1) 460 | (d/datom 1026 :trip/id "foo" :source-2)}))))) 461 | 462 | (testing "PREVIOUS DB, MULTIPLE SOURCES" 463 | (testing "Add and remove across sources" 464 | (let [{:keys [db-after]} (sut/with-sources (db) {:prepare-vessels [{:vessel/imo "123" 465 | :vessel/name "Fyken"} 466 | {:vessel/imo "234" 467 | :vessel/name "Syken"}] 468 | :prepare-services [{:service/id :s123 469 | :service/allocated-vessel {:vessel/imo "123"} 470 | :service/label "fast"} 471 | {:service/id :s234 472 | :service/allocated-vessel {:vessel/imo "234"} 473 | :service/label "slow"}]})] 474 | (assert-report 475 | (sut/with-sources db-after {:prepare-vessels [{:vessel/imo "234" 476 | :vessel/name "Tyken"} 477 | {:vessel/imo "345" 478 | :vessel/name "Titanic"}] 479 | :prepare-services [{:service/id :s123 480 | :service/allocated-vessel {:vessel/imo "123"} 481 | :service/label "fast"} 482 | {:service/id :s234 483 | :service/allocated-vessel {:vessel/imo "345"} 484 | :service/label "slow"}]}) 485 | 486 | {:tx-data [[:db/retract 1024 :vessel/name "Fyken"] 487 | [:db/retract 1025 :vessel/name "Syken"] 488 | [:db/retract 1027 :service/allocated-vessel 1025] 489 | [:db/add 1025 :vessel/name "Tyken"] 490 | [:db/add 1027 :service/allocated-vessel 1028] 491 | [:db/add 1028 :vessel/imo "345"] 492 | [:db/add 1028 :vessel/name "Titanic"]] 493 | :eavs #{(d/datom 1024 :vessel/imo "123" :prepare-services) 494 | (d/datom 1025 :vessel/imo "234" :prepare-vessels) 495 | (d/datom 1025 :vessel/name "Tyken" :prepare-vessels) 496 | (d/datom 1026 :service/allocated-vessel 1024 :prepare-services) 497 | (d/datom 1026 :service/id :s123 :prepare-services) 498 | (d/datom 1026 :service/label "fast" :prepare-services) 499 | (d/datom 1027 :service/allocated-vessel 1028 :prepare-services) 500 | (d/datom 1027 :service/id :s234 :prepare-services) 501 | (d/datom 1027 :service/label "slow" :prepare-services) 502 | (d/datom 1028 :vessel/imo "345" :prepare-services) 503 | (d/datom 1028 :vessel/imo "345" :prepare-vessels) 504 | (d/datom 1028 :vessel/name "Titanic" :prepare-vessels)} 505 | :refs {[:vessel/imo "123"] 1024 506 | [:vessel/imo "234"] 1025 507 | [:service/id :s123] 1026 508 | [:service/id :s234] 1027 509 | [:vessel/imo "345"] 1028}}))))) 510 | 511 | (deftest transact! 512 | (testing "transact! works as with only wrapped with an atom" 513 | (assert-report 514 | (sut/transact! (atom (db)) :dummy [{:vessel/imo "100"}]) 515 | 516 | {:eavs #{(d/datom 1024 :vessel/imo "100" :dummy)} 517 | :refs {[:vessel/imo "100"] 1024}}))) 518 | 519 | (deftest transact-sources! 520 | (testing "transact-sources! works as with-sources only wrapped with an atom" 521 | (assert-report 522 | (sut/transact-sources! (atom (db)) {:dummy [{:vessel/imo "100"}]}) 523 | 524 | {:eavs #{(d/datom 1024 :vessel/imo "100" :dummy)} 525 | :refs {[:vessel/imo "100"] 1024}}))) 526 | 527 | (deftest explode 528 | (testing "explode happy days" 529 | (is (= (sut/explode schema [{:vessel/imo "123" :vessel/name "Fyken"}]) 530 | {:refs {[:vessel/imo "123"] 1024} 531 | :datoms [[1024 :vessel/imo "123"] 532 | [1024 :vessel/name "Fyken"]]}))) 533 | 534 | (testing "explode throws on conflicting values for attribute" 535 | (try (sut/explode schema [{:vessel/imo "123" :vessel/name "Fyken"} 536 | {:vessel/imo "123" :vessel/name "Syken"}]) 537 | (is (= :should-throw :didnt)) 538 | (catch Exception e 539 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 540 | (is (= {:attr :vessel/name 541 | :entity-ref [:vessel/imo "123"] 542 | :conflicting-values #{"Fyken" "Syken"}} 543 | (ex-data e)))))) 544 | 545 | (testing "explode throws on conflicting refs and contains lookup-refs in ex" 546 | (try (sut/explode schema [{:service/id :s567 :service/allocated-vessel {:vessel/imo "123"}} 547 | {:service/id :s567 :service/allocated-vessel {:vessel/imo "456"}}]) 548 | (is (= :should-throw :didnt)) 549 | (catch Exception e 550 | (is (= "Conflicting values asserted for entity" (.getMessage e))) 551 | (is (= {:attr :service/allocated-vessel 552 | :entity-ref [:service/id :s567] 553 | :conflicting-values #{[:vessel/imo "123"] [:vessel/imo "456"]}} 554 | (ex-data e))))))) 555 | 556 | (deftest get-datoms [] 557 | (testing "Get all datoms for db with one source" 558 | (is (= (-> (sut/with (db) :dummy [{:vessel/imo "123" :vessel/name "Fyken"} 559 | {:vessel/imo "234" :vessel/name "Syken"}]) 560 | :db-after 561 | sut/get-datoms) 562 | [[1024 :vessel/imo "123"] 563 | [1024 :vessel/name "Fyken"] 564 | [1025 :vessel/imo "234"] 565 | [1025 :vessel/name "Syken"]]))) 566 | 567 | (testing "Get all datoms for db with multiple sources" 568 | (is (= (-> (sut/with-sources (db) {:source-1 [{:vessel/imo "123" :vessel/name "Fyken"} 569 | {:vessel/imo "234" :vessel/name "Syken"}] 570 | :source-2 [{:service/id :s1 :service/allocated-vessel {:vessel/imo "123"}} 571 | {:service/id :s2 :service/allocated-vessel {:vessel/imo "234"}}]}) 572 | :db-after 573 | sut/get-datoms) 574 | [[1024 :vessel/imo "123"] 575 | [1024 :vessel/name "Fyken"] 576 | [1025 :vessel/imo "234"] 577 | [1025 :vessel/name "Syken"] 578 | [1026 :service/allocated-vessel 1024] 579 | [1026 :service/id :s1] 580 | [1027 :service/allocated-vessel 1025] 581 | [1027 :service/id :s2]])))) 582 | 583 | (deftest export-db 584 | (testing "Export db regression test" 585 | (let [{:keys [db-after]} (sut/with (db) :dummy [{:vessel/imo "123" :vessel/name "Fyken"}])] 586 | (is (= (sut/export-db db-after) 587 | (str "#datascript/DB {" 588 | ":schema " 589 | "{:route/tags #:db{:cardinality :db.cardinality/many}," 590 | " :service/label {}," 591 | " :route/name {}," 592 | " :holiday/bus-key #:db{:unique :db.unique/identity, :tupleAttrs [:holiday/pattern :holiday/weekday]}," 593 | " :my-tuple/id #:db{:unique :db.unique/identity}," 594 | " :holiday/weekday {}," 595 | " :route/holidays #:db{:valueType :db.type/ref, :cardinality :db.cardinality/many}," 596 | " :trip/id #:db{:unique :db.unique/identity}," 597 | " :vessel/imo #:db{:unique :db.unique/identity}," 598 | " :service/trips #:db{:valueType :db.type/ref, :cardinality :db.cardinality/many, :isComponent true}," 599 | " :vessel/name {}," 600 | " :my-tuple/a {}, :my-tuple/tup #:db{:tupleAttrs [:my-tuple/a :my-tuple/b]}," 601 | " :holiday/pattern {}," 602 | " :route/services #:db{:valueType :db.type/ref, :cardinality :db.cardinality/many}," 603 | " :service/allocated-vessel #:db{:valueType :db.type/ref, :cardinality :db.cardinality/one}," 604 | " :route/number #:db{:unique :db.unique/identity}," 605 | " :my-tuple/b {}," 606 | " :service/id #:db{:unique :db.unique/identity}}, " 607 | ":datoms " 608 | "[[1024 :vessel/imo \"123\" 536870912]" 609 | " [1024 :vessel/name \"Fyken\" 536870912]]" 610 | "}")))))) 611 | 612 | (deftest prune-diffs 613 | (testing "Prune diffs regression test" 614 | (is (= (sut/prune-diffs schema 615 | [[:db/retract 2025 :service/allocated-vessel 2026] 616 | [:db/add 2025 :service/allocated-vessel 2027]]) 617 | [[:db/add 2025 :service/allocated-vessel 2027]])))) 618 | --------------------------------------------------------------------------------