├── .travis.yml ├── resources └── icon.png ├── .gitignore ├── test ├── integrity │ ├── walkers_test.clj │ ├── number_test.clj │ ├── test_helpers.clj │ ├── datomic_integration_test.clj │ ├── hal_test.clj │ ├── avro_test.clj │ ├── human_test.clj │ └── datomic_test.clj └── resources │ └── avro-test-schema.json ├── src └── integrity │ ├── walkers.clj │ ├── hal.clj │ ├── number.clj │ ├── avro.clj │ ├── datomic.clj │ └── human.clj ├── project.clj ├── README.md └── LICENSE /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | lein: lein2 3 | script: lein2 do clean, test 4 | -------------------------------------------------------------------------------- /resources/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cddr/integrity/HEAD/resources/icon.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | /doc 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | /.lein-* 10 | /.nrepl-port 11 | -------------------------------------------------------------------------------- /test/integrity/walkers_test.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.walkers-test 2 | (:require [clojure.test :refer :all] 3 | [integrity.walkers :refer [lookup]] 4 | [schema.core :as s :refer [Str]])) 5 | 6 | (deftest walkers-resolve-test 7 | (testing "can resolve a datom" 8 | (let [ticker? {:ticker Str} 9 | ticker (fn [str] {:ticker str}) 10 | stocks {(ticker "AAPL") "apple" 11 | (ticker "GOOG") "google" 12 | (ticker "TWTR") "twitter"} 13 | schema [ticker?]] 14 | (is (= ["apple" "twitter" "google"] 15 | ((lookup schema ticker? #(second (find stocks %))) 16 | [{:ticker "AAPL"} 17 | {:ticker "TWTR"} 18 | {:ticker "GOOG"}])))))) 19 | -------------------------------------------------------------------------------- /src/integrity/walkers.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.walkers 2 | (:require [schema.core :as s])) 3 | 4 | (defn lookup 5 | "Returns a function that takes a single object `data`, and returns a new 6 | object of the same shape with any references replaced with the result of 7 | calling `(lookup-ref data)` 8 | 9 | To determine whether some part of the data is a reference, it checks the 10 | corresponding section of `input-schema` for equality with `reference-type`" 11 | [input-schema reference-type lookup-ref] 12 | (letfn [(walk-fn [s] 13 | (let [walk (s/walker s)] 14 | (fn [data] 15 | (if (= reference-type s) 16 | (lookup-ref data) 17 | (walk data)))))] 18 | (s/start-walker walk-fn input-schema))) 19 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject cddr/integrity "0.3.0-SNAPSHOT" 2 | :description "A collection of libraries for maintaining data integrity" 3 | :url "https://github.com/cddr/integrity" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.7.0"] 7 | [prismatic/schema "0.4.4"] 8 | [com.taoensso/tower "3.1.0-beta4"] 9 | [com.damballa/abracad "0.4.13"]] 10 | :plugins [[codox "0.6.7"]] 11 | :codox {:writer codox-md.writer/write-docs 12 | :output-dir "doc/v0.3.0" 13 | :src-dir-uri "http://github.com/cddr/integrity/blob/master/"} 14 | 15 | 16 | :profiles 17 | {:test {:dependencies [[com.datomic/datomic-free "0.9.5344"]] 18 | :resource-paths ["test/resources"]} 19 | :dev {:dependencies [[com.datomic/datomic-free "0.9.5344"]] 20 | :resource-paths ["test/resources"]}}) 21 | -------------------------------------------------------------------------------- /test/integrity/number_test.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.number-test 2 | (:require [clojure.test :refer :all] 3 | [schema.core :refer [check]] 4 | [integrity.number :refer [gt gte lt lte between]] 5 | [integrity.human :refer [human-walker]])) 6 | 7 | (defn is-valid [test-schema val] 8 | (is (nil? ((human-walker test-schema) val)))) 9 | 10 | (defn is-invalid [test-schema val err] 11 | (is (= err ((human-walker test-schema) val)))) 12 | 13 | (deftest test-lt 14 | (doto (lt 1) 15 | (is-valid 0.9) 16 | (is-invalid 1 "1 is not lt 1"))) 17 | 18 | (deftest test-lte 19 | (doto (lte 1) 20 | (is-valid 1) 21 | (is-invalid 1.5 "1.5 is not lte 1"))) 22 | 23 | (deftest test-gte 24 | (doto (gte 1) 25 | (is-valid 1) 26 | (is-invalid 0.9 "0.9 is not gte 1"))) 27 | 28 | (deftest test-between 29 | (doto (between 1 5) 30 | (is-valid 1.1) 31 | (is-valid 4.9) 32 | (is-invalid 1 "1 is not between 1 and 5") 33 | (is-invalid 5 "5 is not between 1 and 5"))) 34 | 35 | (test-between) 36 | -------------------------------------------------------------------------------- /src/integrity/hal.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.hal 2 | (:require [schema.core :as s :refer [Str Any Bool Keyword]])) 3 | 4 | (def ^{:private true} 5 | opt s/optional-key) 6 | 7 | (def Link 8 | "Returns a schema that matches a HAL Link Object 9 | 10 | See [ietf spec](http://tools.ietf.org/html/draft-kelly-json-hal-06#section-5)" 11 | {:href Str 12 | (opt :templated) Bool 13 | (opt :type) Str 14 | (opt :deprecation) Str 15 | (opt :name) Str 16 | (opt :profile) Str 17 | (opt :title) Str 18 | (opt :hreflang) Str}) 19 | 20 | (def Resource 21 | "Returns a schema that matches a HAL Resource Object 22 | 23 | See [ietf spec](http://tools.ietf.org/html/draft-kelly-json-hal-06#section-4)" 24 | {:_links 25 | {:self Link 26 | Keyword (s/either Link [Link])} 27 | Keyword Any}) 28 | 29 | 30 | (def Curie 31 | "Returns a schema that matches a HAL Curie Object 32 | 33 | See [ietf spec](http://tools.ietf.org/html/draft-kelly-json-hal-06#section-8.2)" 34 | {:name Str 35 | :href Str 36 | (opt :templated) Bool}) 37 | 38 | -------------------------------------------------------------------------------- /test/resources/avro-test-schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "example.com", 3 | "type": "record", 4 | "name": "AvroTestSchema", 5 | "fields": [ 6 | {"name": "boolean", "type": "boolean"}, 7 | {"name": "int", "type": "int"}, 8 | {"name": "long", "type": "long"}, 9 | {"name": "float", "type": "float"}, 10 | {"name": "double", "type": "double"}, 11 | {"name": "bytes", "type": "bytes"}, 12 | {"name": "string", "type": "string"}, 13 | {"name": "record", "type": { 14 | "type": "record", 15 | "name": "rec", 16 | "fields": [ 17 | {"name": "a", "type": "string"}, 18 | {"name": "b", "type": "string"} 19 | ]}}, 20 | {"name": "enum", "type": { 21 | "type": "enum", 22 | "name": "enum", 23 | "symbols": ["a", "b", "c"]}}, 24 | {"name": "array", "type": { 25 | "type": "array", 26 | "name": "array", 27 | "items": "string"}}, 28 | {"name": "map", "type": { 29 | "type": "map", 30 | "name": "map", 31 | "values": "long"}}, 32 | {"name": "fixed", "type": { 33 | "type": "fixed", 34 | "name": "fixed", 35 | "size": 16}} 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /test/integrity/test_helpers.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.test-helpers 2 | (:require [datomic.api :as d])) 3 | 4 | (def ^{:dynamic true} 5 | *prefix* []) 6 | 7 | (defmacro with-prefix [prefix & body] 8 | `(binding [*prefix* (conj *prefix* ~prefix)] 9 | (vec (concat ~@body)))) 10 | 11 | (defn build-ident [ident] 12 | (if (empty? *prefix*) 13 | ident 14 | (let [parts (concat (map (comp str name) (interpose "." *prefix*)) 15 | "/" 16 | (name ident))] 17 | (keyword (apply str parts))))) 18 | 19 | (defn attr [ident type & flags] 20 | (let [enum-idents (:enum-idents type)] 21 | ((comp vec concat) 22 | [{:db/id (d/tempid :db.part/db) 23 | :db/ident (build-ident ident) 24 | :db/valueType (cond 25 | (keyword? type) (keyword (str "db.type/" (name type))) 26 | (:enum-idents type) :db.type/ref) 27 | :db/cardinality (keyword (str "db.cardinality/" 28 | (name (or (first (filter #{:many :one} flags)) 29 | :one)))) 30 | :db.install/_attribute :db.part/db}] 31 | enum-idents))) 32 | -------------------------------------------------------------------------------- /src/integrity/number.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.number 2 | (:require [schema.core :as s] 3 | [schema.utils :as utils] 4 | [integrity.human :refer :all])) 5 | 6 | (defn lt 7 | "Returns a `schema.core` predicate that passes when it's input is less than `high`" 8 | [high] 9 | (s/pred (fn [x] 10 | (< x high)) 11 | `(lt ~high))) 12 | 13 | (defn lte 14 | "Returns a `schema.core` predicate that passes when it's input is less than 15 | or equal to `high`" 16 | [high] 17 | (s/pred (fn [x] 18 | (<= x high)) 19 | `(lte ~high))) 20 | 21 | (defn gt 22 | "Returns a `schema.core` predicate that passes when it's input is greater than `low`" 23 | [low] 24 | (s/pred (fn [x] 25 | (> x low)) 26 | `(gt ~low))) 27 | 28 | (defn gte 29 | "Returns a `schema.core` predicate that passes when it's input is greater than 30 | or equal to `low`" 31 | [low] 32 | (s/pred (fn [x] 33 | (>= x low)) 34 | `(gte ~low))) 35 | 36 | (defn between 37 | "Returns a `schema.core` predicate that passes when it's input is between `low` and `high`" 38 | [low high] 39 | (s/pred (fn [x] 40 | (< low x high)) 41 | `(between ~low ~high))) 42 | 43 | -------------------------------------------------------------------------------- /test/integrity/datomic_integration_test.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.datomic-integration-test 2 | (:require [clojure.test :refer :all] 3 | [schema.core :as s :refer [Str Num Inst Int]] 4 | [datomic.api :as d] 5 | [integrity.datomic :as db])) 6 | 7 | (def test-db-uri "datomic:mem://schema-test") 8 | 9 | (defn list-attrs [db] 10 | (set (map first 11 | (d/q '[:find ?name :where [_ :db.install/attribute ?a] [?a :db/ident ?name]] 12 | db)))) 13 | 14 | (defn db-with [transactions] 15 | (d/create-database test-db-uri) 16 | (let [c (d/connect test-db-uri)] 17 | (:db-after (d/with (d/db c) transactions)))) 18 | 19 | ;; datomic helpers 20 | 21 | (defn installed-attrs [db] 22 | (let [q '[:find ?name 23 | :where [_ :db.install/attribute ?a] 24 | [?a :db/ident ?name]]] 25 | (map (comp #(d/entity db %) first) (d/q q db)))) 26 | 27 | (defn find-attr [db name] 28 | (let [q '[:find ?id 29 | :in $ ?name 30 | :where [_ :db.install/attribute ?id] 31 | [?id :db/ident ?name]]] 32 | (d/entity db (ffirst (d/q q db name))))) 33 | 34 | (def Tweet 35 | {:created-at Inst 36 | :user Str 37 | :msg Str}) 38 | 39 | (deftest schema-test 40 | (testing "can load a generated schema into datomic" 41 | (let [test-db (db-with (db/attributes Tweet {}))] 42 | (is (d/attribute test-db :created-at)) 43 | (is (d/attribute test-db :user)) 44 | (is (d/attribute test-db :msg))))) 45 | -------------------------------------------------------------------------------- /src/integrity/avro.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.avro 2 | "Generates a prismatic schema from an avro one" 3 | (:require 4 | [clojure.java.io :as io] 5 | [schema.core :as s :refer [Bool Str]] 6 | [abracad.avro :as avro]) 7 | (:import [org.apache.avro Schema$Type])) 8 | 9 | (def ByteArray (Class/forName "[B")) 10 | 11 | (defn ->map [avro-schema] 12 | (condp = (.getType avro-schema) 13 | 14 | ;; Primitive types 15 | Schema$Type/BOOLEAN Bool 16 | Schema$Type/INT Integer 17 | Schema$Type/LONG Long 18 | Schema$Type/FLOAT Float 19 | Schema$Type/DOUBLE Double 20 | Schema$Type/BYTES ByteArray 21 | Schema$Type/STRING Str 22 | 23 | ;; Complex Types 24 | Schema$Type/RECORD 25 | (apply hash-map (mapcat (fn [key val] 26 | [(keyword key) val]) 27 | (map #(.name %) (.getFields avro-schema)) 28 | (map #(->map (.schema %)) (.getFields avro-schema)))) 29 | 30 | Schema$Type/ENUM 31 | (apply s/enum (.getEnumSymbols avro-schema)) 32 | 33 | Schema$Type/ARRAY 34 | [(->map (.getElementType avro-schema))] 35 | 36 | Schema$Type/MAP 37 | {Str (->map (.getValueType avro-schema))} 38 | 39 | Schema$Type/FIXED 40 | (s/pred (fn [str-val] 41 | (<= (.getFixedSize avro-schema) (count str-val))) 42 | 'exceeds-fixed-size))) 43 | 44 | (defn avro-errors [schema value] 45 | (let [schema-as-map (->map schema)] 46 | (s/check schema-as-map value))) 47 | 48 | 49 | -------------------------------------------------------------------------------- /test/integrity/hal_test.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.hal-test 2 | (:require [clojure.test :refer :all] 3 | [integrity.hal :as hal] 4 | [schema.core :as s :refer [check]])) 5 | 6 | (deftest hal-resource-tests 7 | (let [self (fn [other-links] 8 | {:_links 9 | (conj 10 | {:self {:href "/foo"}} 11 | other-links)})] 12 | (testing "resource" 13 | (testing "should have link to self" 14 | (is (nil? (check hal/Resource (self nil))))) 15 | 16 | (testing "should permit related links" 17 | (is (nil? (check 18 | hal/Resource 19 | (self {:child {:href "/foo/child"}}))))) 20 | 21 | (testing "should permit arbitrary properties" 22 | (is (nil? (check 23 | hal/Resource 24 | (merge (self nil) 25 | {:bar "baz" 26 | :a-list [1 2 3] 27 | :an-object {:prop "yolo"}})))))))) 28 | 29 | (deftest hal-link-tests 30 | (testing "link" 31 | (is (nil? (check hal/Link {:href "http://example.com"}))) 32 | (is (nil? (check hal/Link {:href "http://example.com" 33 | :templated true 34 | :type "application/hal+json" 35 | :deprecation "http://example/deprected-apis" 36 | :name "cool api 2.0" 37 | :profile "http://example/profile" 38 | :title "A cool resource" 39 | :hreflang "en"}))) 40 | (is (check hal/Link "yolo")) 41 | (is (check hal/Link {:unknown "attr"})))) 42 | 43 | (deftest hal-curie-tests 44 | (testing "curie" 45 | (is (nil? (check hal/Curie {:name "doc" 46 | :href "/doc/{rel}" 47 | :templated true}))) 48 | (is (check hal/Curie "yolo")))) 49 | 50 | (deftest hal-test-unit 51 | (hal-resource-tests) 52 | (hal-link-tests) 53 | (hal-curie-tests)) 54 | -------------------------------------------------------------------------------- /test/integrity/avro_test.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.avro-test 2 | (:require 3 | [abracad.avro :as avro] 4 | [clojure.test :refer :all] 5 | [clojure.java.io :as io] 6 | [schema.core :as s :refer [Str Bool Num Int Inst Keyword]] 7 | [integrity.avro :refer :all] 8 | [integrity.test-helpers :refer [attr]]) 9 | (:import [java.util Date UUID])) 10 | 11 | (def test-schema (->map (avro/parse-schema 12 | (slurp (io/reader 13 | (io/resource "avro-test-schema.json")))))) 14 | 15 | (defn- good? [attr values] 16 | (not-any? #(attr (s/check test-schema %)) 17 | (map (partial hash-map attr) values))) 18 | 19 | (defn- bad? [attr values] 20 | (every? #(attr (s/check test-schema %)) 21 | (map (partial hash-map attr) values))) 22 | 23 | (deftest test-primitive-types 24 | (testing "boolean" 25 | (is (good? :boolean [true false])) 26 | (is (bad? :boolean [42 "42" {}]))) 27 | 28 | (testing "int" 29 | (is (good? :int (map int [0 1 Integer/MAX_VALUE Integer/MIN_VALUE]))) 30 | (is (bad? :int [0.0 (+ Integer/MAX_VALUE 1)]))) 31 | 32 | (testing "long" 33 | (is (good? :long (map long [0 1 Long/MAX_VALUE Long/MIN_VALUE]))) 34 | (is (bad? :long [(+ (bigdec Long/MAX_VALUE) 1) 35 | (- (bigdec Long/MIN_VALUE) 1)]))) 36 | 37 | (testing "float" 38 | (is (good? :float (map float [3.14]))) 39 | (is (bad? :float (map int [3.14])))) 40 | 41 | (testing "double" 42 | (is (good? :double [3.14])) 43 | (is (bad? :double (map int [3.14])))) 44 | 45 | (testing "bytes" 46 | (is (good? :bytes [(.getBytes "hello")])) 47 | (is (bad? :bytes ["hello"]))) 48 | 49 | (testing "string" 50 | (is (good? :string ["abc"])) 51 | (is (bad? :string [42])))) 52 | 53 | (deftest test-complex-types 54 | (testing "record" 55 | (is (good? :record [{:a "a", :b "b"}])) 56 | (is (bad? :record [{:a "missing b"}])) 57 | (is (bad? :record ["not a record"]))) 58 | 59 | (testing "enum" 60 | (is (good? :enum ["a" "b" "c"])) 61 | (is (bad? :enum ["d"]))) 62 | 63 | (testing "array" 64 | (is (good? :array [["first" "second"]])) 65 | (is (bad? :array [[1 2]]))) 66 | 67 | (testing "map" 68 | (is (good? :map [{"one" 1, "two" 2}])) 69 | (is (bad? :map [{"one" 1.0, "two" 2.0}]))) 70 | 71 | (testing "fixed" 72 | (is (good? :fixed ["1234123412341234"])) 73 | (is (bad? :fixed ["1" "123412341234123"])))) 74 | -------------------------------------------------------------------------------- /test/integrity/human_test.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.human-test 2 | (:require [clojure.test :refer :all] 3 | [schema.core :as s :refer [check]] 4 | [integrity.number :refer [gt lt between]] 5 | [integrity.human :refer [human-walker]])) 6 | 7 | (deftest class-explainer 8 | (let [chk (human-walker s/Str)] 9 | (is (nil? (chk "foo"))) 10 | (is (= (chk 42) "42 is not a java.lang.String")))) 11 | 12 | (deftest pred-explainer 13 | ;; since s/Int is implemented as a predicate, we'll just put this here 14 | (let [chk (human-walker s/Int)] 15 | (is (nil? (chk 42))) 16 | (is (= (chk 0.5) "0.5 is not integer")) 17 | (is (= (chk "foo") "foo is not integer"))) 18 | 19 | ;; same for s/Keyword 20 | (let [chk (human-walker s/Keyword)] 21 | (is (nil? (chk :yolo))) 22 | (is (= (chk "yolo") "yolo is not keyword"))) 23 | 24 | ;; TODO: this with more interesting predicates 25 | (let [chk (human-walker (s/pred #(even? %) 26 | 'even?))] 27 | (is (nil? (chk 2))) 28 | (is (= "1 is not even" (chk 1))))) 29 | 30 | (deftest eq-explainer 31 | (let [chk (human-walker (s/eq 42))] 32 | (is (nil? (chk 42))) 33 | (is (= (chk 43) "43 is not eq with 42")))) 34 | 35 | (deftest map-explainer 36 | (let [chk (human-walker {:foo s/Str, :bar s/Str})] 37 | (is (nil? (chk {:foo "foo", :bar "bar"}))) 38 | (is (= (chk {:foo 42, :bar "bar"}) 39 | {:foo "42 is not a java.lang.String"})) 40 | (is (= (chk {:foo "foo", :bar 42}) 41 | {:bar "42 is not a java.lang.String"})))) 42 | 43 | (deftest enum-explainer 44 | (let [chk (human-walker (s/enum 4 3 2))] 45 | (is (nil? (chk 2))) 46 | (is (= (chk 5) "5 is not one of #{4 3 2}")))) 47 | 48 | (deftest nested-schema 49 | (let [chk (human-walker {:name s/Str 50 | :info {:email s/Str}})] 51 | (is (nil? (chk {:name "funk d'void" 52 | :info {:email "void@mixcloud.com"}}))) 53 | (is (= {:name "42 is not a java.lang.String" 54 | :info {:email "42 is not a java.lang.String"}} 55 | (chk {:name 42, :info {:email 42}}))))) 56 | 57 | 58 | ;; TODO: This worked when we implemented the explainer by parsing the output of `check` but 59 | ;; since refactoring to use the human-explain protocol, I can't figure out how to make it 60 | ;; work. The `either` schema construct is discouraged in any case so it's not a priority for 61 | ;; me to fix 62 | ;; 63 | ;; https://groups.google.com/d/msg/prismatic-plumbing/GXKcbM4Ij-Y/BWuDWml42EsJ 64 | ;; 65 | ;; (testing "either" 66 | ;; (is (= "1 fails all of the following:- 67 | ;; it is not a java.lang.String 68 | ;; it is not gt 42 69 | ;; " 70 | ;; (human-explain (check (s/either s/Str 71 | ;; (gt 42)) 72 | ;; 1)))) 73 | ;; (is (= () 74 | ;; (human-explain (check (s/either {:email s/Str} 75 | ;; {:phone s/Str}) 76 | ;; {:email 42})) 77 | -------------------------------------------------------------------------------- /test/integrity/datomic_test.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.datomic-test 2 | (:require [clojure.test :refer :all] 3 | [schema.core :as s :refer [Str Bool Num Int Inst Keyword]] 4 | [datomic.api :as d] 5 | [integrity.datomic :as db] 6 | [integrity.test-helpers :refer [attr]]) 7 | (:import [java.util Date UUID])) 8 | 9 | (def AllTypes 10 | {:a Str 11 | :b Bool 12 | :c Num 13 | :d Int 14 | :e Inst 15 | :f {:nested Bool}}) 16 | 17 | (defn with-test-db [schema uniqueness f] 18 | (let [uri "datomic:mem://test-db"] 19 | (d/create-database uri) 20 | (let [c (d/connect uri) 21 | db (:db-after (d/with (d/db c) (db/attributes schema uniqueness))) 22 | result (f db)] 23 | (d/release c) 24 | (d/delete-database uri) 25 | result))) 26 | 27 | (defn find-attr [db name] 28 | (let [q '[:find ?id 29 | :in $ ?name 30 | :where [_ :db.install/attribute ?id] 31 | [?id :db/ident ?name]]] 32 | (d/entity db (ffirst (d/q q db name))))) 33 | 34 | (deftest test-schema 35 | (with-test-db AllTypes {} 36 | (let [type= (fn [db t attr-name] 37 | (= t (:db/valueType (find-attr db attr-name))))] 38 | (fn [db] 39 | (is (type= db :db.type/string :a)) 40 | (is (type= db :db.type/boolean :b)) 41 | (is (type= db :db.type/double :c)) 42 | (is (type= db :db.type/long :d)) 43 | (is (type= db :db.type/instant :e)) 44 | (is (type= db :db.type/ref :f)) 45 | 46 | (is (type= db :db.type/boolean :nested)))))) 47 | 48 | (deftest test-unique-constraints 49 | (with-test-db AllTypes {:a :db.unique/identity 50 | :b :db.unique/value} 51 | (fn [db] 52 | (is (= :db.unique/identity (:db/unique (find-attr db :a)))) 53 | (is (= :db.unique/value (:db/unique (find-attr db :b))))))) 54 | 55 | ;; (deftest test-simple-map 56 | ;; (let [map-schema {:name Str 57 | ;; :address Str} 58 | ;; uniqueness {:name :db.unique/identity}] 59 | ;; (is (= (set (remove-ids [(db/attribute :name Str :db.unique/identity) 60 | ;; (db/attribute :address Str)])) 61 | ;; (set (remove-ids (db/attributes map-schema uniqueness))))))) 62 | 63 | ;; (deftest test-nested-map 64 | ;; (let [schema {:name Str 65 | ;; :address {:street Str 66 | ;; :city Str}}] 67 | ;; (is (= (set (remove-ids [(db/attribute :name Str) 68 | ;; (db/attribute :street Str) 69 | ;; (db/attribute :city Str) 70 | ;; ((:attr-factory db/Ref) :address)])) 71 | ;; (set (remove-ids (db/attributes schema))))))) 72 | 73 | ;; (def test-datomic-facts 74 | ;; (let [project-data {:name "cddr/integrity" 75 | ;; :url "https://github.com/cddr/integrity" 76 | ;; :license {:name "Eclipse Public License" 77 | ;; :url "http://www.eclipse.org/legal/epl-v10.html"}}] 78 | ;; (db/datomic-facts 0 project-data)) 79 | 80 | -------------------------------------------------------------------------------- /src/integrity/datomic.clj: -------------------------------------------------------------------------------- 1 | (ns integrity.datomic 2 | "Maps datomic attribute definitions to prismatic schemata 3 | and vice versa" 4 | (:require [datomic.api :as d] 5 | [schema.core :as s :refer [Str Num Inst Int Bool Keyword]] 6 | [clojure.walk :as w])) 7 | 8 | (def Ref {:schema (s/either Int Keyword) 9 | :attr-factory (fn [ident] 10 | {:db/id (d/tempid :db.part/db) 11 | :db/ident ident 12 | :db/valueType :db.type/ref 13 | :db/cardinality :db.cardinality/one 14 | :db.install/_attribute :db.part/db})}) 15 | 16 | (def ^{:private true} 17 | schema->datomic 18 | {Str :db.type/string 19 | Bool :db.type/boolean 20 | Long :db.type/long 21 | ;java.Math.BigInteger :db.type/bigint 22 | Num :db.type/double 23 | Int :db.type/long 24 | Float :db.type/float 25 | Inst :db.type/instant 26 | ;java.Math.BigDecimal :db.type/bigdec 27 | }) 28 | 29 | (defmulti attribute 30 | "Implementing methods should return a function that will generate a 31 | datomic attribute when given it's id as the one and only argument" 32 | (fn dispatch 33 | ([ident schema] 34 | (dispatch ident schema false)) 35 | ([ident schema unique] 36 | (class schema)))) 37 | 38 | (defmethod attribute ::leaf 39 | ([ident schema] 40 | (attribute ident schema false)) 41 | ([ident schema unique] 42 | (let [uniqueness (if unique {:db/unique unique} {})] 43 | (merge uniqueness 44 | {:db/id (d/tempid :db.part/db) 45 | :db/ident ident 46 | :db/valueType ((comp val find) schema->datomic schema) 47 | :db/cardinality :db.cardinality/one 48 | :db.install/_attribute :db.part/db})))) 49 | 50 | (defmethod attribute ::vector 51 | ([ident schema] 52 | (attribute ident schema false)) 53 | ([ident schema unique] 54 | (if unique 55 | (throw (Exception. "attributes with cardinality of many cannot be unique")) 56 | {:db/id (d/tempid :db.part/db) 57 | :db/ident ident 58 | :db/valueType ((comp val find) schema->datomic (first schema)) 59 | :db/cardinality :db.cardinality/many 60 | :db.install/_attribute :db.part/db}))) 61 | 62 | (defn attributes 63 | ([schema] 64 | (attributes schema {})) 65 | ([schema uniqueness] 66 | (let [mk-attr (fn [k v] 67 | (attribute k v (k uniqueness)))] 68 | (reduce (fn [acc [k v]] 69 | (if (extends? schema.core/Schema (class v)) 70 | (into acc [(mk-attr k v)]) 71 | (into acc (conj (attributes (into {} v) uniqueness) 72 | ((:attr-factory Ref) k))))) 73 | [] 74 | (seq schema))))) 75 | 76 | (derive java.lang.Class ::leaf) 77 | 78 | ;; This is only required because some single valued Schema types (e.g. Int) 79 | ;; are implemented as predicates. We're not trying to generate datomic attributes 80 | ;; for arbitrary predicates 81 | (derive schema.core.Predicate ::leaf) 82 | 83 | (derive clojure.lang.IPersistentVector ::vector) 84 | -------------------------------------------------------------------------------- /src/integrity/human.clj: -------------------------------------------------------------------------------- 1 | ;; The schema `check` function returns an object which is not really suitable 2 | ;; for displaying an error to an end-user. However, it usually contains enough 3 | ;; information to generate one. This library parses the return value of an invocation 4 | ;; of `check` and generates human readable error messages 5 | 6 | (ns integrity.human 7 | (:require [schema.core :as s] 8 | [schema.utils :as utils] 9 | [taoensso.tower :as tower :refer [*locale*]]) 10 | (:import (schema.utils ValidationError))) 11 | 12 | ;; ### Support for internationalization 13 | 14 | (def ^{:private true} 15 | dictionary 16 | "`dictionary` defines translations so that error messages in multiple 17 | languages can be easily supported" 18 | {:dev-mode? true 19 | :fallback-locale :en 20 | :dictionary 21 | {:en {:integrity.human 22 | {:it "it" 23 | :not-eq "is not eq with" 24 | :not-one-of "is not one of" 25 | :is "is" 26 | :is-not "is not" 27 | :and "and" 28 | :fails-all "fails all of the following:-" 29 | :is-not-a "is not a"}}}}) 30 | 31 | (defn- tval [k] 32 | (tower/t (or *locale* :en) dictionary k)) 33 | 34 | ;; Helpers 35 | (defn- humanize 36 | "`humanize` takes a value and returns a human readable representation 37 | of that value" 38 | [v] 39 | (if (symbol? v) 40 | (clojure.string/replace (str (name v)) "?" "") 41 | v)) 42 | 43 | (defprotocol HumanExplain 44 | (human-explain [schema error] "Explain an error related to this schema")) 45 | 46 | (defn human-walker 47 | [input-schema] 48 | (s/start-walker 49 | (fn [s] 50 | (let [walk (s/walker s)] 51 | (fn [x] 52 | (let [result (walk x)] 53 | (human-explain s result))))) 54 | input-schema)) 55 | 56 | (extend-protocol HumanExplain 57 | java.lang.Class 58 | (human-explain [schema result] 59 | (if (utils/error? result) 60 | (with-out-str 61 | (print (.-value (utils/error-val result)) 62 | (tval ::is-not-a) 63 | schema))))) 64 | 65 | (extend-protocol HumanExplain 66 | schema.core.Predicate 67 | (human-explain [schema result] 68 | (if (utils/error? result) 69 | (let [err (utils/error-val result) 70 | [pred val] @(.-expectation-delay err)] 71 | (with-out-str 72 | (print (.-value err) (tval ::is) (or (.-fail-explanation err) 'not) 73 | (let [name (.-pred-name schema)] 74 | (cond 75 | (symbol? name) (humanize name) 76 | (seq name) (str (humanize (first name)) 77 | " " 78 | (apply str 79 | (interpose (str " " (tval ::and) " ") 80 | (map humanize (rest name))))))))))))) 81 | 82 | (extend-protocol HumanExplain 83 | schema.core.EqSchema 84 | (human-explain [schema result] 85 | (if (utils/error? result) 86 | (let [err (utils/error-val result)] 87 | (with-out-str 88 | (print (.-value err) (tval ::not-eq) (.-v schema))))))) 89 | 90 | (extend-protocol HumanExplain 91 | schema.core.EnumSchema 92 | (human-explain [schema result] 93 | (if (utils/error? result) 94 | (let [err (utils/error-val result)] 95 | (with-out-str 96 | (print (.-value err) (tval ::not-one-of) 97 | (.-vs schema))))))) 98 | 99 | (extend-protocol HumanExplain 100 | schema.core.MapEntry 101 | (human-explain [schema result] 102 | (if-let [err (second result)] 103 | {(first result) err}))) 104 | 105 | (extend-protocol HumanExplain 106 | clojure.lang.PersistentArrayMap 107 | (human-explain [schema result] 108 | (let [m (into {} (map (fn [[schema-key schema-val] res] 109 | (human-explain (s/map-entry schema-key schema-val) res)) 110 | schema result))] 111 | (if (empty? m) 112 | nil 113 | m)))) 114 | 115 | (extend-protocol HumanExplain 116 | schema.core.Either 117 | (human-explain [schema result] 118 | result)) 119 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # integrity 2 | 3 | ![The Schema Toolbox](resources/icon.png) 4 | 5 | The [Schema Project](https://github.com/Prismatic/schema) is aimed at 6 | providing a method of defining the "shape" of your public data structures. 7 | This library is a collection of Schema utilities that we believe may be 8 | useful to a large part of the community. 9 | 10 | ## Status 11 | 12 | * ![Build Status](https://travis-ci.org/cddr/integrity.svg) 13 | 14 | ## Usage 15 | 16 | Add the latest version into your dependencies 17 | 18 | ``` 19 | (defproject 20 | :dependencies [[cddr/integrity "0.3.0-SNAPSHOT"]]) 21 | ``` 22 | 23 | ### integrity.datomic 24 | 25 | The `attributes` function generates datomic attribute definitions to match the specified 26 | schema 27 | 28 | [Datomic Tests](https://github.com/cddr/integrity/blob/master/test/integrity/datomic_test.clj) 29 | 30 | ### integrity.hal 31 | 32 | HAL is the [hypertext application language](http://stateless.co/hal_specification.html). 33 | As the summary at the link above describes, HAL based APIs are easily discoverable by 34 | client applications. The vars in this namespace may be helpful when generating 35 | walkers that require knowledge of HAL data-structures. For example usage, see 36 | the tests 37 | 38 | [HAL Tests](https://github.com/cddr/integrity/blob/master/test/integrity/hal_test.clj) 39 | 40 | ### integrity.human 41 | 42 | When some data input is checked against some schema, `prismatic/schema` 43 | returns a ValidationError object. The `human-explain` function translates 44 | this error object into a message that should be surfaceable to an end-user. 45 | For example usage, see the tests 46 | 47 | [Human Explain Tests](https://github.com/cddr/integrity/blob/master/test/integrity/human_test.clj) 48 | 49 | When using schema's `pred` type constructor, be sure to give your predicate 50 | a name which satisfies the function `human-expectation?`. This should ensure 51 | that the information needed by `ValidationTransformer` to print a human 52 | readable message is attached to your predicate function. 53 | 54 | ### integrity.number 55 | 56 | Schema defines a `pred` utility which builds a schema that matches it's 57 | input if the supplied predicate returns true. Here, we use this to build 58 | numeric schemas that can be more specific than just a type of number. 59 | 60 | For example `(gt 21)` builds a schema one could use to ensure the input 61 | data is old enough to buy booze. For more examples, see the tests 62 | 63 | [Number Tests](https://github.com/cddr/integrity/blob/master/test/integrity/number_test.clj) 64 | 65 | ### integrity.walkers 66 | 67 | Schema walker generators take a schema as input, and use it to return a 68 | function that walks input data in-step with the the corresponding schema. For 69 | example, the `lookup` walker replaces "references" in the input document 70 | with the result of looking them up in an external data source. For example 71 | usage, see the tests 72 | 73 | [Walker Tests](https://github.com/cddr/integrity/blob/master/test/integrity/walkers_test.clj) 74 | 75 | ## Contributing 76 | 77 | If you've found a bug, you have the following options, ordered by usefulness 78 | to the community 79 | 80 | 1. Issue a pull request that contains a test that fails against a released 81 | version, together with a change that fixes the test. 82 | 2. Create an issue describing the minimal steps to reproduce; the version 83 | of the project you are using; what you see; and what you expected to see 84 | 3. Create an issue describing the problem in as much detail as you can 85 | 86 | If you have an idea for a feature, make a github issue and lets talk about it 87 | 88 | ### Running the tests 89 | 90 | Before issuing a pull request, please use the following command to ensure 91 | the unit tests pass and the API docs can be generated 92 | ``` 93 | $ lein do clean, test 94 | ``` 95 | 96 | ## Credits 97 | 98 | [Tree Icon](https://www.iconfinder.com/icons/60170/content_tree_icon#size=256) designed by 99 | [Custom Icon Design](http://www.customicondesign.com) from iconfinder is licensed for 100 | non-commercial use 101 | 102 | ## License 103 | 104 | Copyright © 2014 Andy Chambers 105 | 106 | Distributed under the Eclipse Public License either version 1.0 or (at 107 | your option) any later version. 108 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of Washington and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | --------------------------------------------------------------------------------