├── Dockerfile ├── .gitignore ├── CHANGELOG.md ├── deps.edn ├── config └── logback.xml ├── src └── mysql_to_datomic │ ├── get_mysql.clj │ ├── gen_schema.clj │ ├── core.clj │ ├── gen_spec.clj │ ├── migrations.clj │ └── transform.clj ├── README.md ├── pom.xml └── mysql-to-datomic.iml /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:8-alpine 2 | MAINTAINER Thomas Spellman 3 | 4 | ADD target/mysql-to-datomic.jar /app.jar 5 | 6 | CMD ["java", "-jar", "/app.jar"] 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml.asc 5 | *.jar 6 | *.class 7 | /.lein-* 8 | /.nrepl-port 9 | .hgignore 10 | .hg/ 11 | .idea 12 | logs 13 | private/ 14 | .cpcache 15 | .env 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # [mysql-to-datomic](https://github.com/thosmos/mysql-to-datomic) 2 | 3 | ## [Unreleased] 4 | ### Planned 5 | - more robust FK tracking to enable better interlinked multi-table updates 6 | 7 | ## [0.3.5] - 2020-01-03 8 | ### Fixed 9 | - standalone app usage ... 10 | ### Changed 11 | - improved README and comment code in core.clj 12 | - removed project.clj; uses only deps.edn 13 | 14 | ## [0.3.4] - 2019-02-02 15 | ### Added 16 | - some fns in core for moving new data from mysql into existing matching datomic schema 17 | - use specs for doing some of the migration heavy lifting 18 | - CHANGELOG (so meta ...) 19 | 20 | ## [0.3.3] - 2018-12-02 21 | ### Changed 22 | - updated deps 23 | 24 | ## [0.3.1] - 2018-12-02 25 | - can update an existing migration with new data 26 | - generates and uses a compound-key on many-to-many lookup tables 27 | 28 | ## 0.3.0 - 2018-03-11 29 | - initial commit 30 | - worked for getting a bunch of simple schemas and data into Datomic 31 | 32 | [Unreleased]: https://github.com/thosmos/mysql-to-datomic/compare/0.3.5..HEAD 33 | [0.3.5]: https://github.com/thosmos/mysql-to-datomic/compare/0.3.4...0.3.5 34 | [0.3.4]: https://github.com/thosmos/mysql-to-datomic/compare/0.3.3...0.3.4 35 | [0.3.3]: https://github.com/thosmos/mysql-to-datomic/compare/0.3.1...0.3.3 36 | [0.3.1]: https://github.com/thosmos/mysql-to-datomic/compare/0.3.0...0.3.1 37 | 38 | #### Change Log 39 | All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 40 | 41 | ### Planned 42 | ### Changed 43 | ### Removed 44 | ### Fixed 45 | ### Added 46 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:deps 2 | {org.clojure/clojure {:mvn/version "1.10.1"}, 3 | org.clojure/java.jdbc {:mvn/version "0.7.3"}, 4 | clojure.java-time {:mvn/version "0.3.1"}, 5 | datomic-schema {:mvn/version "1.3.0"}, 6 | org.clojure/tools.logging {:mvn/version "0.4.1"}, 7 | ch.qos.logback/logback-classic {:mvn/version "1.2.3"}, 8 | org.clojure/tools.cli {:mvn/version "0.3.7"}, 9 | com.datomic/datomic-free {:mvn/version "0.9.5697" :exclusions [org.slf4j/slf4j-nop]} 10 | ;com.datomic/datomic-pro {:mvn/version "0.9.5786" :exclusions [org.slf4j/slf4j-nop org.slf4j/jul-to-slf4j 11 | ; org.slf4j/log4j-over-slf4j org.slf4j/jcl-over-slf4j]}, 12 | thosmos/util {:mvn/version "0.1.8"}, 13 | lynxeyes/dotenv {:mvn/version "1.0.2"}, 14 | mysql/mysql-connector-java {:mvn/version "5.1.44"}, 15 | com.rpl/specter {:mvn/version "1.0.4"}, 16 | ;thosmos/domain-spec {:local/root "../domain-spec" :exclusions [com.datomic/datomic-free]}, 17 | thosmos/domain-spec {:mvn/version "0.1.2" :exclusions [com.datomic/datomic-free]}, 18 | hikari-cp {:mvn/version "1.8.1"}, 19 | environ {:mvn/version "1.1.0"}} 20 | 21 | 22 | :paths ["src" "config"] 23 | 24 | :aliases {:repl {:jvm-opts ["-server" "-Dclojure.server.repl={:port,5555,:accept,clojure.core.server/repl}"]} 25 | :low-mem {:jvm-opts ["-Xmx370m" "-Xms128m" "-Ddatomic.objectCacheMax=128m"]} 26 | :run {:main-opts ["-m" "mysql-to-datomic.core"]}} 27 | 28 | :mvn/repos 29 | {"central" {:url "https://repo1.maven.org/maven2/"} 30 | "clojars" {:url "https://clojars.org/repo"} 31 | "my.datomic.com" {:url "https://my.datomic.com/repo"}}} 32 | -------------------------------------------------------------------------------- /config/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 10 | 11 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 12 | 13 | 14 | 15 | 16 | logs/mysql-to-datomic-%d{yyyy-MM-dd}.%i.log 17 | 18 | 64 MB 19 | 20 | 21 | 22 | true 23 | 24 | 25 | INFO 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | %-5level %logger{36} - %msg%n 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /src/mysql_to_datomic/get_mysql.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-to-datomic.get-mysql 2 | (:require 3 | [clojure.tools.logging :as log :refer [debug info warn error]] 4 | [clojure.string :as s :refer [join]] 5 | [clojure.java.jdbc :as j] 6 | [clojure.pprint :refer [pprint]] 7 | [com.rpl.specter :refer :all] 8 | [thosmos.util :refer [merge-tree]])) 9 | 10 | 11 | (defn get-tables [meta] 12 | (map :table_name 13 | (j/metadata-result 14 | (.getTables meta nil nil nil (into-array String ["TABLE"]))))) 15 | 16 | (defn get-columns [meta table] 17 | (map #(select-keys % [:column_name 18 | :type_name 19 | :data_type 20 | :column_size 21 | :decimal_digits 22 | :nullable 23 | :is_autoincrement 24 | :ordinal_position]) 25 | (j/metadata-result 26 | (.getColumns meta nil nil table nil)))) 27 | 28 | 29 | (defn into-map [key coll] 30 | (into {} 31 | (for [c coll] 32 | [(keyword (s/replace (get c key) #" " "_")) c]))) 33 | 34 | (defn get-columns-map [meta table] 35 | (into-map :column_name (get-columns meta table))) 36 | 37 | (defn get-primary-keys [meta table] 38 | (vec 39 | (map :column_name 40 | (sort-by :key_seq 41 | (j/metadata-result 42 | (.getPrimaryKeys meta nil nil table)))))) 43 | 44 | (defn get-foreign-keys [meta table] 45 | (map #(select-keys % [:pktable_name :pkcolumn_name 46 | :fktable_name :fkcolumn_name 47 | :delete_rule :update_rule]) 48 | (j/metadata-result 49 | (.getImportedKeys meta nil nil table)))) 50 | 51 | (defn get-foreign-keys-map [meta table] 52 | (into-map :fkcolumn_name (get-foreign-keys meta table))) 53 | 54 | 55 | (defn add-rev-map-keys [tables] 56 | (into {} 57 | (for [[k table] tables] 58 | (let [v (assoc table :rev-keys 59 | (into-map :fktable_name 60 | (select [MAP-VALS :foreign-keys MAP-VALS #(= (:pktable_name %) (name k))] tables)))] 61 | [k v])))) 62 | 63 | (defn get-indexes [meta table] 64 | (j/metadata-result 65 | (.getIndexInfo meta nil nil table false false))) 66 | 67 | (defn tablator [mysql-db] 68 | (j/with-db-metadata [meta mysql-db] 69 | (let [tables (get-tables meta) 70 | tables (remove #(clojure.string/ends-with? % "_entry") tables) 71 | tables (into (sorted-map) 72 | (for [table tables] 73 | [(keyword table) {:columns (get-columns-map meta table) 74 | :foreign-keys (get-foreign-keys-map meta table) 75 | :primary-keys (get-primary-keys meta table)}])) 76 | 77 | tables (add-rev-map-keys tables)] 78 | 79 | tables))) 80 | -------------------------------------------------------------------------------- /src/mysql_to_datomic/gen_schema.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-to-datomic.gen-schema 2 | (:require 3 | [clojure.tools.logging :as log :refer [debug info warn error]] 4 | [datomic-schema.schema :as s])) 5 | 6 | ;; The main schema functions 7 | (defn fields 8 | "Simply a helper for converting (fields [name :string :indexed]) into {:fields {\"name\" [:string #{:indexed}]}} 9 | Modified from the macros from datomic-schema.schema" 10 | [fielddefs] 11 | (let [defs (reduce (fn [a [nm tp & opts]] (assoc a (name nm) [tp (set opts)])) {} fielddefs)] 12 | {:fields defs})) 13 | 14 | (defn schema 15 | "Simply merges several maps into a single schema definition and add one or two helper properties. 16 | Modified from the macros from datomic-schema.schema" 17 | [nm maps] 18 | (apply merge 19 | {:name (name nm) :basetype (keyword nm) :namespace (name nm)} 20 | maps)) 21 | 22 | (defn partinator [part] [(s/part part)]) 23 | 24 | 25 | (defn schemalator [state tables-maps] 26 | (for [[t tm] tables-maps] 27 | (let [flds (for [[f fm] (:columns tm)] 28 | (let [result [f] 29 | fk (get-in tm [:foreign-keys f]) 30 | pks (get tm :primary-keys) 31 | ;sql-t (:type_name fm) 32 | type (case (:type_name fm) 33 | "VARCHAR" :string 34 | "LONGTEXT" :string 35 | "TEXT" :string 36 | "CHAR" :string 37 | "BIT" :boolean 38 | "INT" :long 39 | "SMALLINT" :long 40 | "INT UNSIGNED" :long 41 | "TINYINT UNSIGNED" :long 42 | "DOUBLE" :double 43 | "DECIMAL" :bigdec 44 | "DATETIME" :instant 45 | "DATE" :instant 46 | "TIMESTAMP" :instant 47 | "TIME" :instant 48 | :string) 49 | type (if fk :ref type) 50 | _ (swap! state assoc-in [:types t f] type) 51 | result (conj result type) 52 | result (cond-> result 53 | (not= type :string) 54 | (conj :indexed) 55 | 56 | (and (= 1 (count pks)) (= (first pks) (name f))) 57 | (conj :unique-identity))] 58 | 59 | 60 | result))] 61 | (schema t 62 | (fields flds))))) 63 | 64 | (defn generator [state part table-maps] 65 | (concat 66 | (s/generate-parts (partinator part)) 67 | (s/generate-schema (schemalator state table-maps)))) 68 | 69 | 70 | (defn spec-schemalator [specs] 71 | (debug "convert mysql schema into model specs") 72 | 73 | (vec 74 | (concat 75 | [[:global/uuid :one :uuid :identity "a globally unique ID"]] 76 | (apply concat 77 | (for [{:keys [entity/attrs entity/pks entity/name]} specs] 78 | (vec 79 | (for [{:keys [attr/key attr/cardinality attr/type attr/primary? attr/unique? attr/identity? 80 | attr/toggles attr/doc]} attrs] 81 | (vec 82 | (concat 83 | [key cardinality type] 84 | 85 | (when identity? 86 | [:identity]) 87 | 88 | ;(when primary? 89 | ; [:index]) 90 | 91 | (when (and (not= type :string) (not= type :ref)) 92 | [:index]) 93 | 94 | (when unique? 95 | [:unique]) 96 | 97 | toggles 98 | 99 | [(or doc "")]))))))))) 100 | 101 | 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MySQL-to-Datomic 2 | [![Clojars Project](https://img.shields.io/clojars/v/thosmos/mysql-to-datomic.svg)](https://clojars.org/thosmos/mysql-to-datomic) 3 | 4 | A Clojure library designed to automatically transfer data from MySQL to Datomic. 5 | 6 | It analyzes the schema of the MySQL DB, including single-field foreign keys, 7 | generates a Datomic schema, transforms the data from MySQL into Datomic format, and transacts it. 8 | 9 | 10 | ### Run as an app 11 | 12 | This depends on some environment variables: 13 | ``` 14 | MYSQL_DATABASE 15 | MYSQL_USERNAME 16 | MYSQL_PASSWORD 17 | MYSQL_HOST (default 127.0.0.1) 18 | MYSQL_PORT (default 3306) 19 | DATOMIC_URI (default "datomic:mem://mysql-datomic-test1") 20 | ``` 21 | 22 | LIKE: 23 | ```bash 24 | clojure -A:run 25 | ``` 26 | 27 | OR POSSIBLY LIKE?: 28 | ```bash 29 | MYSQL_DATABASE=mydb MYSQL_USERNAME=myuser clojure -A:run 30 | ``` 31 | 32 | OR MAYBE LIKE?: 33 | ```bash 34 | clojure -J-Dmysql.database=riverdb -J-Dmysql.username=riverdb -A:run 35 | ``` 36 | 37 | ### From the REPL 38 | 39 | Start a repl like: 40 | 41 | ```bash 42 | clj 43 | ``` 44 | 45 | The function `mysql-to-datomic.core/run-conversion` will do the full process: 46 | 47 | ```clojure 48 | (require 'mysql-to-datomic.core) 49 | (in-ns 'mysql-to-datomic.core) 50 | (def uri "datomic:sql://newdb?jdbc:mysql://localhost:3306/datomic?user=datomic&password=datomic&useSSL=false") 51 | (d/create-database uri) 52 | (def cx (d/connect uri)) 53 | (def my-ds (create-mysql-datasource {:mysql-username "myuser" 54 | :mysql-password "mypass" 55 | :mysql-database "mydb"})) 56 | (def tx (run-conversion state cx my-ds)) 57 | ``` 58 | 59 | or in more detail allowing doing other things with the generated pieces: 60 | 61 | ```clojure 62 | (require 'mysql-to-datomic.core) 63 | (in-ns 'mysql-to-datomic.core) 64 | 65 | (def uri "datomic:sql://newdb?jdbc:mysql://localhost:3306/datomic?user=datomic&password=datomic&useSSL=false") 66 | (def cx (d/connect uri)) 67 | (defn db [] (d/db cx)) 68 | 69 | (def mydb (create-mysql-datasource {:mysql-username "myuser" 70 | :mysql-password "mypass" 71 | :mysql-database "mydb"})) 72 | (def tables (get-mysql/tablator mydb)) 73 | (def specs (gen-spec/tables->specs tables)) 74 | 75 | ;; put the specs into a datascript DB for querying 76 | (def dsdb (domain-spec.core/new-specs-ds)) 77 | (ds/transact dsdb specs) 78 | 79 | ;;;; generate terse DB schemas from the specs 80 | (def d-schema-terse (gen-schema/spec-schemalator specs)) 81 | 82 | ;;;; generate datascript schemas 83 | (schema-tx-ds d-schema-terse) 84 | 85 | ;;;; generate datomic schemas 86 | (schema-tx d-schema-terse) 87 | 88 | ;;;; transact the data into a temp DB 89 | (def txw (transform/run-main-fields-with (db) state my-ds specs tables)) 90 | (def txw' (transform/run-fks-with (:db-after txs) state tables)) 91 | (:db-after txw') 92 | 93 | ;;;; transact it for real ... careful 94 | (def tx (transform/run-main-fields cx state my-ds specs tables)) 95 | (def tx' (transform/run-fks cx state tables)) 96 | ``` 97 | 98 | 99 | ## JVM Resource Usage 100 | 101 | It has been somewhat optimized to run in a low memory virtual machine. You will probably need to 102 | tweak the JVM memory settings though. 103 | 104 | `datomic.objectCacheMax` should match your transactor's equivalent setting. 105 | 106 | ```bash 107 | clojure -J-server -J-Xmx370m -J-Xms128m -J-Ddatomic.objectCacheMax=128m -A:run 108 | ``` 109 | 110 | ### NOTE 111 | 112 | It currently only handles the MySQL data types that I needed. 113 | The code will most likely need some tweaking for your use case. 114 | Feel free to create an issue or pull request and I'll be happy to help. 115 | 116 | ## License 117 | 118 | Copyright © 2017 Thomas Spellman 119 | 120 | Distributed under the Eclipse Public License either version 1.0 or (at 121 | your option) any later version. 122 | -------------------------------------------------------------------------------- /src/mysql_to_datomic/core.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-to-datomic.core 2 | (:gen-class) 3 | (:require 4 | [clojure.java.jdbc :as j] 5 | [clojure.tools.logging :as log :refer [debug info warn error]] 6 | [mysql-to-datomic.get-mysql :as get-mysql] 7 | [mysql-to-datomic.gen-schema :as gen-schema] 8 | [mysql-to-datomic.transform :as transform] 9 | [mysql-to-datomic.gen-spec :as gen-spec] 10 | [mysql-to-datomic.migrations :as mig] 11 | [datomic.api :as d] 12 | [clojure.java.io :as io] 13 | [datascript.core :as ds] 14 | [environ.core :refer [env]] 15 | [clojure.data :refer [diff]] 16 | [hikari-cp.core :refer [make-datasource close-datasource]] 17 | [domain-spec.literals :refer [schema-tx-ds schema-tx]] 18 | [domain-spec.core])) 19 | 20 | (defn datasource-options 21 | [opts] 22 | {:minimum-idle 2 23 | :maximum-pool-size 10 24 | :pool-name "db-pool" 25 | :adapter "mysql" 26 | :username (or (:mysql-username opts) (env :mysql-username)) 27 | :password (or (:mysql-password opts) (env :mysql-password)) 28 | :database-name (or (:mysql-database opts) (env :mysql-database)) 29 | :server-name (or (:mysql-host opts) (env :mysql-host) "127.0.0.1") 30 | :port-number (or (:mysql-port opts) (env :mysql-port) 3306) 31 | :use-ssl false}) 32 | 33 | (defn uri [] (or (env :datomic-uri) "datomic:mem://mysql-datomic-test")) 34 | 35 | (defonce state (atom {:tables nil 36 | :fks nil 37 | :pks nil 38 | :tempids nil})) 39 | 40 | (defn create-mysql-datasource [opts] 41 | {:datasource (make-datasource (datasource-options opts))}) 42 | 43 | (defn run-conversion [state cx my-ds] 44 | (let [_ (debug "generating mysql table infos") 45 | tables (get-mysql/tablator my-ds) 46 | 47 | _ (debug "generating table specs") 48 | specs (gen-spec/tables->specs tables) 49 | 50 | _ (debug "transferring main fields") 51 | _ (transform/run-main-fields cx state my-ds specs tables) 52 | 53 | _ (debug "transferring fk fields") 54 | tx (transform/run-fks cx state tables)] 55 | 56 | (debug "Done!") 57 | tx)) 58 | 59 | (defn -main [] 60 | (debug "Running ...") 61 | (let [_ (d/create-database (uri)) 62 | cx (d/connect (uri)) 63 | my-ds (create-mysql-datasource nil) 64 | tx (run-conversion state cx my-ds)] 65 | (swap! state assoc :db-after (:db-after tx)))) 66 | 67 | 68 | 69 | 70 | (comment 71 | 72 | (def uri "datomic:free://localhost:4334/riverdb") 73 | (defn cx [] 74 | (d/connect uri)) 75 | (defn db [] (d/db (cx))) 76 | 77 | (def mydb {:datasource (make-datasource (datasource-options {:mysql-username "myuser" 78 | :mysql-password "mypass" 79 | :mysql-database "mydb"}))}) 80 | (def tables (get-mysql/tablator mydb)) 81 | (def specs (gen-spec/tables->specs tables)) 82 | 83 | ;; put the specs into a datascript DB for querying 84 | (def dsdb (domain-spec.core/new-specs-ds)) 85 | (ds/transact dsdb specs) 86 | 87 | ;;;; generate terse DB schemas from the specs 88 | (def d-schema-terse (gen-schema/spec-schemalator specs)) 89 | 90 | ;;;; generate datascript schemas 91 | (schema-tx-ds d-schema-terse) 92 | 93 | ;;;; generate datomic schemas 94 | (schema-tx d-schema-terse) 95 | 96 | 97 | ;;;; transact the data into a temp DB 98 | (def txw (transform/run-main-fields-with (db) state my-ds specs tables)) 99 | (def txw' (transform/run-fks-with (:db-after txs) state tables)) 100 | (:db-after txw') 101 | 102 | ;;;; transact it for real ... careful 103 | (def tx (transform/run-main-fields cx state my-ds specs tables)) 104 | (def tx' (transform/run-fks cx state tables)) 105 | 106 | ;; clear the process state 107 | (swap! state 108 | #(-> % 109 | (assoc :pks nil) 110 | (assoc :fks nil) 111 | (assoc :tables nil) 112 | (assoc :tempids nil)))) 113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /src/mysql_to_datomic/gen_spec.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-to-datomic.gen-spec 2 | (:require 3 | [clojure.tools.logging :as log :refer [debug info warn error]])) 4 | 5 | (defn tables->specs [tables] 6 | (vec 7 | (for [[table-key table-map] tables] 8 | (let [table-name (name table-key) 9 | fks (:foreign-keys table-map) 10 | pks (:primary-keys table-map) 11 | compound-id? (when 12 | (> (count pks) 1) 13 | (keyword table-name "compound-key")) 14 | 15 | attrs (vec 16 | (for [[col-k col-m] (:columns table-map)] 17 | (let [col-k-nm (name col-k) 18 | attr-key (keyword table-name col-k-nm) 19 | 20 | ;; is this a foreign-key? 21 | fk-m? (filter 22 | (fn [[_ fk-m]] 23 | (= (:fkcolumn_name fk-m) col-k-nm)) 24 | fks) 25 | 26 | fk? (seq fk-m?) 27 | 28 | ;; is this a primary key? 29 | pk? (some #{col-k-nm} pks) 30 | 31 | id? (and 32 | pk? 33 | (= 1 (count pks))) 34 | 35 | typ (condp some [(:type_name col-m)] 36 | #{"BIT"} :boolean 37 | #{"INT" "SMALLINT" "INT UNSIGNED" "TINYINT UNSIGNED"} :long 38 | #{"DOUBLE"} :double 39 | #{"DECIMAL"} :bigdec 40 | #{"DATETIME" "TIMESTAMP" "DATE" "TIME"} :instant 41 | :string)] 42 | 43 | 44 | (merge 45 | {:attr/name col-k-nm 46 | :attr/key attr-key 47 | :attr/position (:ordinal_position col-m)} 48 | 49 | (when pk? 50 | {:attr/primary? true}) 51 | 52 | (when id? 53 | {:attr/identity? true}) 54 | 55 | (when (= 0 (:nullabe col-m)) 56 | {:attr/required? true}) 57 | 58 | (if fk? 59 | {:attr/cardinality :one 60 | :attr/type :ref 61 | :attr/ref {:entity/ns 62 | (keyword "entity.ns" (:pktable_name (last (first fk-m?))))}} 63 | {:attr/cardinality :one 64 | :attr/type typ}) 65 | 66 | (when (= typ :double) 67 | {:attr/decimals (:decimal_digits col-m)}) 68 | 69 | (when (= typ :string) 70 | {:attr/strlen (:column_size col-m)}))))) 71 | 72 | attrs (if compound-id? 73 | (conj attrs 74 | {:attr/name "compound-key" 75 | :attr/key compound-id? 76 | :attr/identity? true 77 | :attr/cardinality :one 78 | :attr/type :string 79 | :attr/doc (str "a unique identity key to join multiple primary key values: " pks)}) 80 | attrs) 81 | 82 | 83 | result {:db/id table-name 84 | :entity/name table-name 85 | :entity/ns (keyword "entity.ns" table-name) 86 | :entity/pks (mapv #(keyword table-name %) pks) 87 | :entity/pr-keys (mapv #(keyword table-name %) pks) 88 | :entity/attrs attrs} 89 | 90 | result (if compound-id? 91 | (assoc result :entity/compound-key compound-id?) 92 | result)] 93 | result)))) 94 | 95 | 96 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | thosmos 5 | mysql-to-datomic 6 | 0.3.6 7 | mysql-to-datomic 8 | A minimal library for moving schema and data from MySQL to Datomic 9 | https://github.com/thosmos/mysql-to-datomic 10 | 11 | 12 | Eclipse Public License 13 | http://www.eclipse.org/legal/epl-v10.html 14 | 15 | 16 | 17 | https://github.com/thosmos/mysql-to-datomic 18 | scm:git:git://github.com/thosmos/mysql-to-datomic.git 19 | scm:git:ssh://git@github.com/thosmos/mysql-to-datomic.git 20 | 21 | 22 | 23 | clojars 24 | Clojars repository 25 | https://clojars.org/repo 26 | 27 | 28 | 29 | 30 | org.clojure 31 | clojure 32 | 1.10.1 33 | 34 | 35 | org.clojure 36 | tools.logging 37 | 0.4.1 38 | 39 | 40 | org.clojure 41 | tools.cli 42 | 0.3.7 43 | 44 | 45 | datomic-schema 46 | datomic-schema 47 | 1.3.0 48 | 49 | 50 | com.datomic 51 | datomic-pro 52 | 0.9.5786 53 | 54 | 55 | org.slf4j 56 | slf4j-nop 57 | 58 | 59 | org.slf4j 60 | jul-to-slf4j 61 | 62 | 63 | org.slf4j 64 | log4j-over-slf4j 65 | 66 | 67 | org.slf4j 68 | jcl-over-slf4j 69 | 70 | 71 | 72 | 73 | thosmos 74 | util 75 | 0.1.8 76 | 77 | 78 | environ 79 | environ 80 | 1.1.0 81 | 82 | 83 | lynxeyes 84 | dotenv 85 | 1.0.2 86 | 87 | 88 | clojure.java-time 89 | clojure.java-time 90 | 0.3.1 91 | 92 | 93 | org.clojure 94 | java.jdbc 95 | 0.7.3 96 | 97 | 98 | mysql 99 | mysql-connector-java 100 | 5.1.44 101 | 102 | 103 | hikari-cp 104 | hikari-cp 105 | 1.8.1 106 | 107 | 108 | thosmos 109 | domain-spec 110 | 0.1.2 111 | 112 | 113 | com.datomic 114 | datomic-free 115 | 116 | 117 | 118 | 119 | com.rpl 120 | specter 121 | 1.0.4 122 | 123 | 124 | ch.qos.logback 125 | logback-classic 126 | 1.2.3 127 | 128 | 129 | 130 | src 131 | 132 | 133 | src 134 | 135 | 136 | 137 | 138 | 139 | clojars 140 | https://clojars.org/repo 141 | 142 | 143 | my.datomic.com 144 | https://my.datomic.com/repo 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /mysql-to-datomic.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /src/mysql_to_datomic/migrations.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-to-datomic.migrations 2 | (:require 3 | [clojure.tools.logging :as log :refer [debug info warn error]] 4 | [datomic.api :as d] 5 | [clojure.java.jdbc :as j])) 6 | 7 | 8 | (defn find-attr-max [db attr] 9 | (d/q '[:find (max ?v) . 10 | :in $ ?attr 11 | :where [_ ?attr ?v]] 12 | db attr)) 13 | 14 | (defn get-pk-maxes [db tables-map table-keys] 15 | (into {} 16 | (for [t-k table-keys] 17 | (let [t-nm (name t-k) 18 | t-map (get tables-map t-k) 19 | pk-kw (keyword t-nm (first (:primary-keys t-map))) 20 | pk-max (find-attr-max db pk-kw)] 21 | [pk-kw pk-max])))) 22 | 23 | (defn migrate-table-txes [mydb tables-map table-key pk-maxes] 24 | (let [table (get tables-map table-key) 25 | fks (:foreign-keys table) 26 | t-nm (name table-key) 27 | sql (str "SELECT * FROM `" t-nm "`") 28 | pks (:primary-keys table) 29 | multi? (> (count pks) 1) 30 | multi-fn (fn [row] 31 | (assoc row (keyword t-nm "compound-key") 32 | (apply str 33 | (map 34 | #(str % 35 | (if-let [fk-m (get fks (keyword %))] 36 | (let [{:keys [fktable_name fkcolumn_name pktable_name pkcolumn_name]} fk-m 37 | fk-kw (keyword fktable_name fkcolumn_name) 38 | fk-val (get row fk-kw) 39 | pk-kw (keyword pktable_name pkcolumn_name) 40 | fk-val (if-let [fk-max (get pk-maxes pk-kw)] 41 | (+ fk-val fk-max) 42 | fk-val)] 43 | fk-val) 44 | (get row (keyword t-nm %)))) 45 | pks)))) 46 | pk-kw (keyword t-nm (first (:primary-keys table))) 47 | pk-max (get pk-maxes pk-kw) 48 | ;; query mysql and add namespaces to all field names 49 | results (j/query mydb sql {:identifiers #(keyword t-nm (name %))})] 50 | (vec 51 | (for [result results] 52 | (let [pk-val (get result pk-kw) 53 | result (cond 54 | multi? 55 | (multi-fn result) 56 | pk-max 57 | (assoc result pk-kw (+ pk-val pk-max)) 58 | :else 59 | result) 60 | result (into {} (remove (fn [[k v]] (nil? v)) result)) 61 | result (assoc result :riverdb.entity/ns (keyword "entity.ns" t-nm))] 62 | (reduce-kv 63 | (fn [result k {:keys [fktable_name fkcolumn_name pktable_name pkcolumn_name]}] 64 | (let [fk-kw (keyword fktable_name fkcolumn_name) 65 | fk-val (get result fk-kw) 66 | pk-kw (keyword pktable_name pkcolumn_name) 67 | fk-val (if-let [fk-max (get pk-maxes pk-kw)] 68 | (+ fk-val fk-max) 69 | fk-val) 70 | fk-ref [pk-kw fk-val]] 71 | (if fk-val 72 | (assoc result fk-kw fk-ref) 73 | result))) 74 | result fks)))))) 75 | 76 | 77 | (defn migrate-tables-txes 78 | ([mydb db tables-map table-keys] 79 | (let [pk-maxes (get-pk-maxes db tables-map table-keys)] 80 | (migrate-tables-txes mydb db tables-map pk-maxes))) 81 | ([mydb db tables-map table-keys pk-maxes] 82 | (for [t-key table-keys] 83 | (migrate-table-txes mydb tables-map t-key pk-maxes)))) 84 | 85 | 86 | (comment 87 | ;;;;;;; migrate new data that has PK values that were reset to 0 to a few tables that already exist in the destination, 88 | ;;;;;;; so we need to add the new PK values to the existing highest PK values 89 | 90 | ;; get primary key values for subsequent migration of select tables 91 | (get-pk-maxes (d/db cx) tables [:sitevisit :sample :fieldresult :labresult :fieldobsresult]) 92 | => {:sitevisit/SiteVisitID 84650, 93 | :sample/SampleRowID 184944, 94 | :fieldresult/FieldResultRowID 1438179, 95 | :labresult/LabResultRowID 108537, 96 | :fieldobsresult/FieldObsResultRowID 164283} 97 | 98 | ;; double check 99 | (ffirst (migrate-tables-txes mydb (d/db cx) tables 100 | [:sitevisit :sample :fieldresult :labresult :fieldobsresult] 101 | {:sitevisit/SiteVisitID 84650, 102 | :sample/SampleRowID 184944, 103 | :fieldresult/FieldResultRowID 1438179, 104 | :labresult/LabResultRowID 108537, 105 | :fieldobsresult/FieldObsResultRowID 164283})) 106 | 107 | ;; do it for real 108 | (for [tx (migrate-tables-txes mydb (d/db cx) tables 109 | [:sitevisit :sample :fieldresult :labresult :fieldobsresult] 110 | {:sitevisit/SiteVisitID 84650, 111 | :sample/SampleRowID 184944, 112 | :fieldresult/FieldResultRowID 1438179, 113 | :labresult/LabResultRowID 108537, 114 | :fieldobsresult/FieldObsResultRowID 164283})] 115 | (count (:tx-data @(d/transact cx tx)))) 116 | => (18025 9974 96709 1 17797)) 117 | 118 | 119 | -------------------------------------------------------------------------------- /src/mysql_to_datomic/transform.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-to-datomic.transform 2 | (:require 3 | [clojure.tools.logging :as log :refer [debug info warn error]] 4 | [clojure.string :refer [join]] 5 | [clojure.java.jdbc :as j] 6 | [clojure.pprint :refer [pprint]] 7 | [datomic.api :as d])) 8 | 9 | (defn rand-str [len] 10 | (apply str (take len (repeatedly #(char (+ (rand 26) 65)))))) 11 | 12 | (defn ns-kw [ns kw] 13 | (keyword (str ns "/" (name kw)))) 14 | 15 | ;;; impure function updates state with foreign-key infos 16 | (defn main-fields-from-specs! [state mydb specs tables] 17 | (remove empty? 18 | (flatten 19 | (remove empty? 20 | (for [{:keys [db/id entity/pks entity/compound-key entity/attrs]} specs] 21 | (let [t-key (keyword id) 22 | table-map (get tables t-key) 23 | t-nm id 24 | _ (debug "processing " t-nm) 25 | ;; get all fields from SQL 26 | sql (str "SELECT * FROM `" t-nm "`") 27 | 28 | ;; query mysql and add namespaces to all field names 29 | results (j/query mydb sql {:identifiers #(keyword (str t-nm "/" (name %)))})] 30 | 31 | ;; make a primary tempid, save foreign keys for later in :fks and remove them, 32 | ;; calc all reverse referenced keys, give them tempids, and save in a lookup table :pks 33 | 34 | (for [r results] 35 | (let [ 36 | ;; remove nil values 37 | r (into {} (remove (fn [[_ v]] (= v nil)) r)) 38 | r (if (empty? r) nil r)] 39 | 40 | (when r 41 | (let [ 42 | ;; generate a tempid string of PK name and its value 43 | dbid (when (seq pks) 44 | (apply str 45 | (map #(str (name %) (get r %)) 46 | pks))) 47 | 48 | r (if 49 | compound-key 50 | (assoc r compound-key dbid) 51 | r) 52 | 53 | ;; if dbid exists, add it, otherwise let datomic generate one as we won't use it 54 | r (assoc r :db/id (if dbid dbid (str t-nm (rand-str 37)))) 55 | 56 | 57 | ;; save minimal records with :db/id and FKs to assert after all records are transacted 58 | fks (:foreign-keys table-map) 59 | fkks (for [[fk _] fks] 60 | (ns-kw t-nm fk)) 61 | _ (when (seq fkks) 62 | (swap! state update-in [:fks t-nm] 63 | conj 64 | (select-keys r (concat [:db/id] fkks)))) 65 | 66 | ;; remove foreign keys to avoid asserting mysql lookup values 67 | r (reduce-kv 68 | (fn [r fk _] 69 | (let [fkk (ns-kw t-nm fk)] 70 | (dissoc r fkk))) 71 | r fks) 72 | 73 | rks (:rev-keys table-map) 74 | ;; get a list of the various primary key names other tables reference 75 | rk-pks (into #{} (for [[_ rm] rks] 76 | (:pkcolumn_name rm)))] 77 | 78 | ;; set the various Rev-PK and their lookup values into state along with the correct dbid 79 | (doseq [rpk rk-pks] 80 | (let [rkk (ns-kw t-nm rpk) 81 | rpk-val (get r rkk)] 82 | (swap! state assoc-in [:pks t-nm rpk rpk-val] dbid))) 83 | 84 | r)))))))))) 85 | 86 | 87 | (defn run-txes-with [db txes] 88 | (let [parts (partition 100 100 nil txes)] 89 | (loop [dbwith db 90 | parts parts 91 | t-ids {}] 92 | (let [part (vec (first parts)) 93 | tx (try 94 | (d/with dbwith part) 95 | (catch Exception ex (do 96 | ;(pprint part) 97 | (pprint ex) 98 | (throw ex)))) 99 | t-ids (merge t-ids (:tempids tx)) 100 | db-after (:db-after tx) 101 | next (next parts)] 102 | (print ".") 103 | (flush) 104 | (if next 105 | (recur db-after next t-ids) 106 | {:db-before db :db-after db-after :tempids t-ids}))))) 107 | 108 | (defn run-txes [cx txes] 109 | (let [db-before (d/db cx) 110 | parts (partition 100 100 nil txes)] 111 | (loop [parts parts 112 | t-ids {}] 113 | (let [part (vec (first parts)) 114 | tx (try 115 | @(d/transact cx part) 116 | (catch Exception ex (do 117 | ;(pprint part) 118 | (pprint ex) 119 | (throw ex)))) 120 | t-ids (merge t-ids (:tempids tx)) 121 | next (next parts)] 122 | (print ".") 123 | (flush) 124 | (if next 125 | (recur next t-ids) 126 | {:db-before db-before :db-after (:db-after tx) :tempids t-ids}))))) 127 | 128 | (defn run-main-fields-with [db state my-db specs tables] 129 | (swap! state 130 | #(-> % 131 | (assoc :pks nil) 132 | (assoc :fks nil) 133 | (assoc :tempids nil))) 134 | (let [tx (run-txes-with db (main-fields-from-specs! state my-db specs tables)) 135 | tempids (:tempids tx)] 136 | (swap! state assoc :tempids tempids) 137 | tx)) 138 | 139 | (defn run-main-fields [cx state my-db specs tables] 140 | (swap! state 141 | #(-> % 142 | (assoc :pks nil) 143 | (assoc :fks nil) 144 | (assoc :tempids nil))) 145 | (let [tx (run-txes cx (main-fields-from-specs! state my-db specs tables)) 146 | tempids (:tempids tx)] 147 | (swap! state assoc :tempids tempids) 148 | tx)) 149 | 150 | (defn foreign-keys [state tables] 151 | ;; loop through our saved foreign key records by table 152 | (let [{:keys [fks pks tempids]} @state] 153 | (flatten 154 | (for [[t-nm fk-rs] fks] 155 | (let [tk (keyword t-nm) 156 | tm (get tables tk) 157 | t-fks (:foreign-keys tm)] 158 | (debug "processing " t-nm) 159 | ;; loop through rows 160 | (for [r fk-rs] 161 | ;; loop through fields 162 | (let [;; replace tempid with final one 163 | dbid-str (:db/id r) 164 | dbid (get tempids dbid-str) 165 | ;; replace :db/id with new eid value 166 | r (assoc r :db/id dbid) 167 | ;; loop through FKs and lookup new ref values 168 | r (into {} 169 | (for [[fkk fk-val] r] 170 | (if (= fkk :db/id) 171 | [fkk fk-val] 172 | (let [;; get the foreign key field name 173 | fk-nm (name fkk) 174 | fk (keyword fk-nm) 175 | 176 | ;; get the foreign key info map 177 | fk-m (get t-fks fk) 178 | 179 | ;; get the primary key table and field name info 180 | pk-table (:pktable_name fk-m) 181 | pk-col (:pkcolumn_name fk-m) 182 | 183 | ;; get the tempid from the primary key lookup table 184 | pk-tempid (get-in pks [pk-table pk-col fk-val]) 185 | 186 | ;; get the eid of the referenced entity from :tempids lookup table 187 | pk-dbid (get tempids pk-tempid)] 188 | 189 | ;; return a MapEntry with the new value 190 | [fkk pk-dbid]))))] 191 | r))))))) 192 | 193 | (defn run-fks-with [db state tables] 194 | (run-txes-with db (foreign-keys state tables))) 195 | 196 | (defn run-fks [cx state tables] 197 | (run-txes cx (foreign-keys state tables))) 198 | 199 | --------------------------------------------------------------------------------