├── CHANGELOG.adoc ├── doc └── intro.md ├── project.clj ├── README.adoc ├── test └── crux_datascript │ └── core_test.clj ├── src └── crux_datascript │ └── core.clj └── LICENSE /CHANGELOG.adoc: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to crux-datascript 2 | 3 | TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) 4 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject crux-datascript "0.1.0-SNAPSHOT" 2 | :description "FIXME: write description" 3 | :url "http://example.com/FIXME" 4 | :license {:name "EPL-2.0" 5 | :url "https://www.eclipse.org/legal/epl-2.0/"} 6 | :dependencies [[org.clojure/clojure "1.10.0"] 7 | [juxt/crux-core "20.01-1.6.3-alpha-SNAPSHOT"] 8 | [datascript "0.18.8"]] 9 | :repl-options {:init-ns crux-datascript.core}) 10 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | = crux-datascript 2 | 3 | A Clojure library designed to ... well, that part is up to you. 4 | 5 | == Usage 6 | 7 | FIXME 8 | 9 | === License 10 | 11 | Copyright © 2020 JUXT 12 | 13 | This program and the accompanying materials are made available under the 14 | terms of the Eclipse Public License 2.0 which is available at 15 | http://www.eclipse.org/legal/epl-2.0. 16 | 17 | This Source Code may also be made available under the following Secondary 18 | Licenses when the conditions for such availability set forth in the Eclipse 19 | Public License, v. 2.0 are satisfied: GNU General Public License as published by 20 | the Free Software Foundation, either version 2 of the License, or (at your 21 | option) any later version, with the GNU Classpath Exception which is available 22 | at https://www.gnu.org/software/classpath/license.html. 23 | -------------------------------------------------------------------------------- /test/crux_datascript/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns crux-datascript.core-test 2 | (:require [clojure.test :refer :all] 3 | [crux-datascript.core :as x] 4 | [datascript.core :as d] 5 | [crux.api :as c])) 6 | 7 | (deftest a-test 8 | (testing "Basic end-to-end" 9 | (let [my-ops [[:crux.tx/put {:crux.db/id :test :a :vall} #inst "2020-01-22" #inst "2020-04-27"] 10 | ;;[:crux.tx/cas {:crux.db/id :test :a :vall} {:crux.db/id :test :a :val2}] 11 | [:crux.tx/put {:crux.db/id :test3 :a :val :what :is-happening :r.blah/bloo :asdf}] 12 | [:crux.tx/delete :test2]] 13 | crux-options 14 | {:crux.node/topology :crux.standalone/topology 15 | :crux.node/kv-store "crux.kv.memdb/kv" ; in-memory, see docs for LMDB/RocksDB storage 16 | :crux.standalone/event-log-kv-store "crux.kv.memdb/kv" ; same as above 17 | :crux.standalone/event-log-dir "data/event-log-dir-1" ; :event-log-dir is ignored when using MemKv 18 | :crux.kv/db-dir "data/db-dir-1"}] ; :db-dir is ignored when using MemKv 19 | 20 | (with-open [node (c/start-node crux-options)] 21 | (when-let [transact-result (x/crux-transact! node my-ops)] 22 | (let [ds-node (d/create-conn {})] 23 | (reset! ds-node 24 | (apply x/crux->datascript (c/db node) @ds-node transact-result)) 25 | (is (= (d/q '[:find ?n ?a 26 | :where 27 | [?n :a :val] 28 | [?n :r.blah/bloo ?b] 29 | [?b :crux.db/id ?a]] 30 | @ds-node) 31 | #{[3 :asdf]})))))))) 32 | -------------------------------------------------------------------------------- /src/crux_datascript/core.clj: -------------------------------------------------------------------------------- 1 | (ns crux-datascript.core 2 | (:require [datascript.core :as d] 3 | [crux.api :as c])) 4 | 5 | 6 | ;; all entities need a :crux.db/id 7 | ;; no cardinality-one attributes 8 | ;; ref attributes have to be prefixed with `r.` 9 | ;; a delete in Crux does not equate to a retractEntity 10 | 11 | ;; transaction-time (tt) is the global `now` for the system, across crux and in datascript, not valid-time (vt) 12 | ;; designed for vt=tt sync with datascript, so vt is used for datascript's tt (if tracked) 13 | ;; this is currently from one node, but what is needed is a feed from all transactions (submitted across all nodes, via hooks on tx-log successes), also combined with the future-time `now` index/polling merged in to support proactive ops (retroactive vt ops would still be out of scope) 14 | 15 | (defn ref-keyword? [k] 16 | (when-let [nspc (namespace k)] 17 | (clojure.string/starts-with? nspc "r."))) 18 | 19 | (defn docs->schema [docs] 20 | (merge (reduce #(assoc %1 %2 {:db/cardinality :db.cardinality/many}) {} (mapcat keys docs)) 21 | (reduce #(assoc %1 %2 {:db/valueType :db.type/ref 22 | :db/cardinality :db.cardinality/many}) {} 23 | (remove #(not (ref-keyword? %)) 24 | (mapcat keys docs))) 25 | {:crux.db/id {:db/unique :db.unique/identity}})) 26 | 27 | (defn doc->lookup-doc [doc] 28 | (into {} 29 | (map (fn [k] 30 | [k (let [v (k doc)] 31 | (if (ref-keyword? k) 32 | [:crux.db/id v] 33 | v))]) 34 | (keys doc)))) 35 | 36 | (defn eid->retractions [db eid] 37 | (let [eid (:db/id (d/entity db eid))] ;; resolve eid if it is a lookup vector 38 | (->> (take-while #(= (:e %) eid) (d/datoms db :eavt eid)) 39 | (filter #(not (contains? #{:db/id :crux.db/id} (:a %)))) 40 | (mapv #(vector :db/retract (:e %) (:a %) (:v %)))))) 41 | 42 | (defn crux-transact! [node ops] 43 | (let [submitted-tx (c/submit-tx node ops)] 44 | (c/await-tx node submitted-tx) 45 | (when (c/tx-committed? node submitted-tx) 46 | [ops submitted-tx]))) 47 | 48 | (defn submitted-tx->hybrid-report [db ops {:keys [crux.tx/tx-id crux.tx/tx-time] :as submitted-tx}] 49 | {:db-after submitted-tx 50 | :db-before (assoc submitted-tx :crux.tx/tx-id (dec tx-id)) ;; good enough? 51 | :tx-time tx-time 52 | :tx-data (reduce into 53 | (mapv (fn [[op & args]] 54 | (let [doc ((if (= op :crux.tx/cas) second first) args) 55 | retract-op [:db.fn/call 56 | eid->retractions 57 | [:crux.db/id (:crux.db/id doc)]]] 58 | (condp contains? op 59 | #{:crux.tx/put :crux.tx/cas} 60 | (into (vec (map #(do {:crux.db/id (% doc)}) 61 | (filter ref-keyword? (keys doc)))) 62 | [retract-op 63 | (doc->lookup-doc (c/entity db (:crux.db/id doc)))]) 64 | #{:crux.tx/delete :crux.tx/evict} 65 | (when (nil? (c/entity db (:crux.db/id doc))) 66 | [retract-op]) 67 | (throw (Exception. (str "crux op not recognised: " op))) 68 | ))) 69 | ops))}) ;; maybe add :tempids and :tx-meta 70 | 71 | (defn crux->datascript [db ds ops submitted-tx] 72 | (let [{:keys [tx-data]} (submitted-tx->hybrid-report db ops submitted-tx) 73 | ds-schema (merge (:schema ds) 74 | (docs->schema (filter map? tx-data)))] 75 | (:db-after (d/with (if (not (= ds-schema (:schema ds))) ;; ds schema migration isn't cheap 76 | (if (nil? (d/datoms ds :eavt)) 77 | @(d/create-conn ds-schema) 78 | @(d/conn-from-datoms (d/datoms ds :eavt) ds-schema)) 79 | ds) 80 | tx-data)))) 81 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Eclipse Public License - v 2.0 2 | 3 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE 4 | PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION 5 | OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 6 | 7 | 1. DEFINITIONS 8 | 9 | "Contribution" means: 10 | 11 | a) in the case of the initial Contributor, the initial content 12 | Distributed under this Agreement, and 13 | 14 | b) in the case of each subsequent Contributor: 15 | i) changes to the Program, and 16 | ii) additions to the Program; 17 | where such changes and/or additions to the Program originate from 18 | and are Distributed by that particular Contributor. A Contribution 19 | "originates" from a Contributor if it was added to the Program by 20 | such Contributor itself or anyone acting on such Contributor's behalf. 21 | Contributions do not include changes or additions to the Program that 22 | are not Modified Works. 23 | 24 | "Contributor" means any person or entity that Distributes the Program. 25 | 26 | "Licensed Patents" mean patent claims licensable by a Contributor which 27 | are necessarily infringed by the use or sale of its Contribution alone 28 | or when combined with the Program. 29 | 30 | "Program" means the Contributions Distributed in accordance with this 31 | Agreement. 32 | 33 | "Recipient" means anyone who receives the Program under this Agreement 34 | or any Secondary License (as applicable), including Contributors. 35 | 36 | "Derivative Works" shall mean any work, whether in Source Code or other 37 | form, that is based on (or derived from) the Program and for which the 38 | editorial revisions, annotations, elaborations, or other modifications 39 | represent, as a whole, an original work of authorship. 40 | 41 | "Modified Works" shall mean any work in Source Code or other form that 42 | results from an addition to, deletion from, or modification of the 43 | contents of the Program, including, for purposes of clarity any new file 44 | in Source Code form that contains any contents of the Program. Modified 45 | Works shall not include works that contain only declarations, 46 | interfaces, types, classes, structures, or files of the Program solely 47 | in each case in order to link to, bind by name, or subclass the Program 48 | or Modified Works thereof. 49 | 50 | "Distribute" means the acts of a) distributing or b) making available 51 | in any manner that enables the transfer of a copy. 52 | 53 | "Source Code" means the form of a Program preferred for making 54 | modifications, including but not limited to software source code, 55 | documentation source, and configuration files. 56 | 57 | "Secondary License" means either the GNU General Public License, 58 | Version 2.0, or any later versions of that license, including any 59 | exceptions or additional permissions as identified by the initial 60 | Contributor. 61 | 62 | 2. GRANT OF RIGHTS 63 | 64 | a) Subject to the terms of this Agreement, each Contributor hereby 65 | grants Recipient a non-exclusive, worldwide, royalty-free copyright 66 | license to reproduce, prepare Derivative Works of, publicly display, 67 | publicly perform, Distribute and sublicense the Contribution of such 68 | Contributor, if any, and such Derivative Works. 69 | 70 | b) Subject to the terms of this Agreement, each Contributor hereby 71 | grants Recipient a non-exclusive, worldwide, royalty-free patent 72 | license under Licensed Patents to make, use, sell, offer to sell, 73 | import and otherwise transfer the Contribution of such Contributor, 74 | if any, in Source Code or other form. This patent license shall 75 | apply to the combination of the Contribution and the Program if, at 76 | the time the Contribution is added by the Contributor, such addition 77 | of the Contribution causes such combination to be covered by the 78 | Licensed Patents. The patent license shall not apply to any other 79 | combinations which include the Contribution. No hardware per se is 80 | licensed hereunder. 81 | 82 | c) Recipient understands that although each Contributor grants the 83 | licenses to its Contributions set forth herein, no assurances are 84 | provided by any Contributor that the Program does not infringe the 85 | patent or other intellectual property rights of any other entity. 86 | Each Contributor disclaims any liability to Recipient for claims 87 | brought by any other entity based on infringement of intellectual 88 | property rights or otherwise. As a condition to exercising the 89 | rights and licenses granted hereunder, each Recipient hereby 90 | assumes sole responsibility to secure any other intellectual 91 | property rights needed, if any. For example, if a third party 92 | patent license is required to allow Recipient to Distribute the 93 | Program, it is Recipient's responsibility to acquire that license 94 | before distributing the Program. 95 | 96 | d) Each Contributor represents that to its knowledge it has 97 | sufficient copyright rights in its Contribution, if any, to grant 98 | the copyright license set forth in this Agreement. 99 | 100 | e) Notwithstanding the terms of any Secondary License, no 101 | Contributor makes additional grants to any Recipient (other than 102 | those set forth in this Agreement) as a result of such Recipient's 103 | receipt of the Program under the terms of a Secondary License 104 | (if permitted under the terms of Section 3). 105 | 106 | 3. REQUIREMENTS 107 | 108 | 3.1 If a Contributor Distributes the Program in any form, then: 109 | 110 | a) the Program must also be made available as Source Code, in 111 | accordance with section 3.2, and the Contributor must accompany 112 | the Program with a statement that the Source Code for the Program 113 | is available under this Agreement, and informs Recipients how to 114 | obtain it in a reasonable manner on or through a medium customarily 115 | used for software exchange; and 116 | 117 | b) the Contributor may Distribute the Program under a license 118 | different than this Agreement, provided that such license: 119 | i) effectively disclaims on behalf of all other Contributors all 120 | warranties and conditions, express and implied, including 121 | warranties or conditions of title and non-infringement, and 122 | implied warranties or conditions of merchantability and fitness 123 | for a particular purpose; 124 | 125 | ii) effectively excludes on behalf of all other Contributors all 126 | liability for damages, including direct, indirect, special, 127 | incidental and consequential damages, such as lost profits; 128 | 129 | iii) does not attempt to limit or alter the recipients' rights 130 | in the Source Code under section 3.2; and 131 | 132 | iv) requires any subsequent distribution of the Program by any 133 | party to be under a license that satisfies the requirements 134 | of this section 3. 135 | 136 | 3.2 When the Program is Distributed as Source Code: 137 | 138 | a) it must be made available under this Agreement, or if the 139 | Program (i) is combined with other material in a separate file or 140 | files made available under a Secondary License, and (ii) the initial 141 | Contributor attached to the Source Code the notice described in 142 | Exhibit A of this Agreement, then the Program may be made available 143 | under the terms of such Secondary Licenses, and 144 | 145 | b) a copy of this Agreement must be included with each copy of 146 | the Program. 147 | 148 | 3.3 Contributors may not remove or alter any copyright, patent, 149 | trademark, attribution notices, disclaimers of warranty, or limitations 150 | of liability ("notices") contained within the Program from any copy of 151 | the Program which they Distribute, provided that Contributors may add 152 | their own appropriate notices. 153 | 154 | 4. COMMERCIAL DISTRIBUTION 155 | 156 | Commercial distributors of software may accept certain responsibilities 157 | with respect to end users, business partners and the like. While this 158 | license is intended to facilitate the commercial use of the Program, 159 | the Contributor who includes the Program in a commercial product 160 | offering should do so in a manner which does not create potential 161 | liability for other Contributors. Therefore, if a Contributor includes 162 | the Program in a commercial product offering, such Contributor 163 | ("Commercial Contributor") hereby agrees to defend and indemnify every 164 | other Contributor ("Indemnified Contributor") against any losses, 165 | damages and costs (collectively "Losses") arising from claims, lawsuits 166 | and other legal actions brought by a third party against the Indemnified 167 | Contributor to the extent caused by the acts or omissions of such 168 | Commercial Contributor in connection with its distribution of the Program 169 | in a commercial product offering. The obligations in this section do not 170 | apply to any claims or Losses relating to any actual or alleged 171 | intellectual property infringement. In order to qualify, an Indemnified 172 | Contributor must: a) promptly notify the Commercial Contributor in 173 | writing of such claim, and b) allow the Commercial Contributor to control, 174 | and cooperate with the Commercial Contributor in, the defense and any 175 | related settlement negotiations. The Indemnified Contributor may 176 | participate in any such claim at its own expense. 177 | 178 | For example, a Contributor might include the Program in a commercial 179 | product offering, Product X. That Contributor is then a Commercial 180 | Contributor. If that Commercial Contributor then makes performance 181 | claims, or offers warranties related to Product X, those performance 182 | claims and warranties are such Commercial Contributor's responsibility 183 | alone. Under this section, the Commercial Contributor would have to 184 | defend claims against the other Contributors related to those performance 185 | claims and warranties, and if a court requires any other Contributor to 186 | pay any damages as a result, the Commercial Contributor must pay 187 | those damages. 188 | 189 | 5. NO WARRANTY 190 | 191 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT 192 | PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" 193 | BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 194 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF 195 | TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR 196 | PURPOSE. Each Recipient is solely responsible for determining the 197 | appropriateness of using and distributing the Program and assumes all 198 | risks associated with its exercise of rights under this Agreement, 199 | including but not limited to the risks and costs of program errors, 200 | compliance with applicable laws, damage to or loss of data, programs 201 | or equipment, and unavailability or interruption of operations. 202 | 203 | 6. DISCLAIMER OF LIABILITY 204 | 205 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT 206 | PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS 207 | SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 208 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST 209 | PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 210 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 211 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 212 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE 213 | POSSIBILITY OF SUCH DAMAGES. 214 | 215 | 7. GENERAL 216 | 217 | If any provision of this Agreement is invalid or unenforceable under 218 | applicable law, it shall not affect the validity or enforceability of 219 | the remainder of the terms of this Agreement, and without further 220 | action by the parties hereto, such provision shall be reformed to the 221 | minimum extent necessary to make such provision valid and enforceable. 222 | 223 | If Recipient institutes patent litigation against any entity 224 | (including a cross-claim or counterclaim in a lawsuit) alleging that the 225 | Program itself (excluding combinations of the Program with other software 226 | or hardware) infringes such Recipient's patent(s), then such Recipient's 227 | rights granted under Section 2(b) shall terminate as of the date such 228 | litigation is filed. 229 | 230 | All Recipient's rights under this Agreement shall terminate if it 231 | fails to comply with any of the material terms or conditions of this 232 | Agreement and does not cure such failure in a reasonable period of 233 | time after becoming aware of such noncompliance. If all Recipient's 234 | rights under this Agreement terminate, Recipient agrees to cease use 235 | and distribution of the Program as soon as reasonably practicable. 236 | However, Recipient's obligations under this Agreement and any licenses 237 | granted by Recipient relating to the Program shall continue and survive. 238 | 239 | Everyone is permitted to copy and distribute copies of this Agreement, 240 | but in order to avoid inconsistency the Agreement is copyrighted and 241 | may only be modified in the following manner. The Agreement Steward 242 | reserves the right to publish new versions (including revisions) of 243 | this Agreement from time to time. No one other than the Agreement 244 | Steward has the right to modify this Agreement. The Eclipse Foundation 245 | is the initial Agreement Steward. The Eclipse Foundation may assign the 246 | responsibility to serve as the Agreement Steward to a suitable separate 247 | entity. Each new version of the Agreement will be given a distinguishing 248 | version number. The Program (including Contributions) may always be 249 | Distributed subject to the version of the Agreement under which it was 250 | received. In addition, after a new version of the Agreement is published, 251 | Contributor may elect to Distribute the Program (including its 252 | Contributions) under the new version. 253 | 254 | Except as expressly stated in Sections 2(a) and 2(b) above, Recipient 255 | receives no rights or licenses to the intellectual property of any 256 | Contributor under this Agreement, whether expressly, by implication, 257 | estoppel or otherwise. All rights in the Program not expressly granted 258 | under this Agreement are reserved. Nothing in this Agreement is intended 259 | to be enforceable by any entity that is not a Contributor or Recipient. 260 | No third-party beneficiary rights are created under this Agreement. 261 | 262 | Exhibit A - Form of Secondary Licenses Notice 263 | 264 | "This Source Code may also be made available under the following 265 | Secondary Licenses when the conditions for such availability set forth 266 | in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), 267 | version(s), and exceptions or additional permissions here}." 268 | 269 | Simply including a copy of this Agreement, including this Exhibit A 270 | is not sufficient to license the Source Code under Secondary Licenses. 271 | 272 | If it is not possible or desirable to put the notice in a particular 273 | file, then You may include the notice in a location (such as a LICENSE 274 | file in a relevant directory) where a recipient would be likely to 275 | look for such a notice. 276 | 277 | You may add additional accurate notices of copyright ownership. 278 | --------------------------------------------------------------------------------