├── .gitignore ├── README.md ├── project.clj └── src ├── dev └── clojure │ └── eginez │ └── huckleberry │ └── runner.cljs ├── main └── clojure │ └── eginez │ └── huckleberry │ ├── core.cljs │ └── os.cljs └── test └── clojure └── eginez └── huckleberry ├── core_test.cljs └── test_runner.cljs /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | *jar 3 | /lib/ 4 | /classes/ 5 | .lein-deps-sum 6 | resources/public/cljs/ 7 | node_modules/ 8 | out/ 9 | .idea 10 | huckleberry.iml 11 | target/ 12 | .lein-repl-history 13 | figwheel_server.log 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Huckleberry [![Clojars Project](https://img.shields.io/clojars/v/org.clojars.eginez/huckleberry.svg)](https://clojars.org/org.clojars.eginez/huckleberry) 2 | A clojurescript library that provides dependency resolution for maven artifacts. 3 | Huckleberry aims to be a jvm-less replacement for [Pomergranate](https://github.com/cemerick/pomegranate) and [Aether](https://github.com/sonatype/sonatype-aether), where possible. 4 | 5 | ## Huckleberry supports 6 | * Maven dependencies expressed in lein style coordinates eg: [commons-logging "1.0"] 7 | * Local repo 8 | * Exclusions 9 | * Resolving transient dependencies via the parent or using versions interpolated from the properties in the POM file where required. 10 | 11 | ## Huckleberry does not support 12 | * Proxies or Mirrors 13 | * Managed coordinates 14 | * Classpath arithmetic/handling 15 | 16 | ## Installation 17 | Huckleberry can be found in clojars. Add it to your leiningen project 18 | ```clojure 19 | [org.clojars.eginez/huckleberry "0.2.0"] 20 | ``` 21 | 22 | ## Usage 23 | The entry function can be used like so 24 | ```clojure 25 | (huckleberry/resolve-depedencies :coordinates '[[commons-logging "1.1" :retrieve false]]) 26 | ``` 27 | This will return a channel which will push a list with [status depedency-graph flatten-depdency-list] 28 | 29 | ```clojure 30 | (huckleberry/resolve-depedencies :coordinates '[[commons-logging "1.1" :retrieve true]]) 31 | ``` 32 | Will return channel that will output the status of each of the files that need to be downloaded 33 | 34 | For more examples on how to use the library look in the test [directory](https://github.com/eginez/huckleberry/blob/master/src/test/clojure/eginez/huckleberry/core_test.cljs) 35 | 36 | ## Running the tests 37 | 38 | Run `lein deps` followed by `lein doo node test`. 39 | 40 | ## License 41 | 42 | Copyright (C) 2016 Esteban Ginez 43 | 44 | Distributed under the Eclipse Public License, the same as Clojure. 45 | 46 | 47 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject org.clojars.eginez/huckleberry "0.2.1" 2 | :url "https://github.com/eginez/huckleberry" 3 | :description "maven dependecy resolution in clojurescript" 4 | :min-lein-version "2.5.3" 5 | :license {:name "Eclipse Public License" 6 | :url "http://www.eclipse.org/legal/epl-v10.html"} 7 | 8 | :dependencies [[org.clojure/clojure "1.8.0"] 9 | [org.clojure/clojurescript "1.9.293"] 10 | [andare "0.4.0"]] 11 | 12 | :plugins [[lein-cljsbuild "1.1.3"] 13 | [lein-cljfmt "0.5.3"] 14 | 15 | [lein-npm "0.6.2"]] 16 | 17 | :source-paths ["src/main/clojure"] 18 | :test-paths ["src/test/clojure" "src/main/clojure"] 19 | 20 | :clean-targets ["main.js" "out" "target"] 21 | 22 | :main "out/main.js" ;; downloads npm deps and run 23 | 24 | :npm { 25 | :dependencies [[xml2js "0.4.17"] 26 | [request "2.74.0"]] 27 | 28 | } 29 | 30 | :cljsbuild { 31 | :builds [ 32 | {:id "dev" 33 | :source-paths ["src/main/clojure" "src/dev/clojure"] 34 | :figwheel true 35 | :compiler { 36 | :main eginez.huckleberry.runner 37 | :output-to "out/main.js" 38 | :target :nodejs 39 | :output-dir "out" 40 | :optimizations :none 41 | :parallel-build true 42 | :source-map true}} 43 | {:id "test" 44 | :source-paths[ "src/main/clojure" "src/test/clojure"] 45 | :compiler { 46 | :main eginez.huckleberry.test-runner 47 | :output-to "out/test.js" 48 | :target :nodejs 49 | :output-dir "out/test" 50 | :optimizations :none 51 | :parallel-build true 52 | :source-map true} 53 | } 54 | ]} 55 | :figwheel {} 56 | :profiles {:dev {:dependencies [[com.cemerick/piggieback "0.2.1"] 57 | [org.clojure/tools.nrepl "0.2.12"]] 58 | :plugins [[lein-doo "0.1.7"] 59 | [lein-figwheel "0.5.8" :exclusions [cider/cider-nrepl]]] 60 | :npm {:dependencies [[ws "1.1.1"]]}}}) 61 | -------------------------------------------------------------------------------- /src/dev/clojure/eginez/huckleberry/runner.cljs: -------------------------------------------------------------------------------- 1 | (ns ^:figwheel-always eginez.huckleberry.runner 2 | (:require [cljs.nodejs :as nodejs])) 3 | 4 | (nodejs/enable-util-print!) 5 | 6 | (defn -main [] 7 | (println "Huckleberry runner started!")) 8 | 9 | (set! cljs.core/*main-cli-fn* -main) 10 | -------------------------------------------------------------------------------- /src/main/clojure/eginez/huckleberry/core.cljs: -------------------------------------------------------------------------------- 1 | (ns eginez.huckleberry.core 2 | (:require-macros [cljs.core.async.macros :refer [go-loop go]]) 3 | (:refer-clojure :exclude [type proxy]) 4 | (:require [cljs.nodejs :as nodejs] 5 | [cljs.core.async :refer [timeout close! put! chan ! take! pipeline alts! poll!] :as async] 6 | [clojure.set :as set] 7 | [eginez.huckleberry.os :as os] 8 | [clojure.string :as str])) 9 | 10 | (def default-repos {:clojars "https://clojars.org/repo" 11 | :local (str/join os/SEPARATOR [os/HOME-DIR ".m2" "repository"]) 12 | :maven-central "https://repo1.maven.org/maven2"}) 13 | 14 | (defn is-url-local? [url] 15 | (not (str/starts-with? url "http"))) 16 | 17 | (defn create-remote-url-for-depedency [repo {group :group artifact :artifact version :version}] 18 | (let [sep (if (is-url-local? repo) "/" os/SEPARATOR) 19 | g (str/replace group #"\." sep) 20 | art (str/join "-" [artifact version]) 21 | art-url (str/join sep [repo g artifact version art]) 22 | ext ["pom" "jar"]] 23 | [repo (map #(str/join "." [art-url %]) ext)])) 24 | 25 | 26 | (defn create-urls-for-dependency [repos d] 27 | (if (coll? repos) 28 | (map #(create-remote-url-for-depedency % d) repos) 29 | (create-remote-url-for-depedency repos d))) 30 | 31 | (defn read-url-chan [cout url] 32 | (if (is-url-local? url) 33 | (os/read-file cout url) 34 | (os/make-http-request cout url))) 35 | 36 | (defn mvndep->dep [x] 37 | (let [g (first (:groupId x)) 38 | a (first (:artifactId x)) 39 | v (first (:version x)) 40 | m {:group g :artifact a :version v}] 41 | m)) 42 | 43 | (defn dep->path[x] 44 | (let [[r [pom jar]] (create-remote-url-for-depedency (:url x) x)] 45 | jar)) 46 | 47 | 48 | (defn dep->coordinate [dep] 49 | (str (:group dep) "/" (:artifact dep) " " (:version dep))) 50 | 51 | (defn clean-deps [x] 52 | (let [ y (remove #(or 53 | (= "test" (first (:scope %))) 54 | (= "true" (first (:optional %))) 55 | (nil? (:version %))) 56 | x)] 57 | y)) 58 | 59 | (defn extract-deps [{:keys [project] :as parsed-xml}] 60 | (let [properties-lookup (->> project 61 | :properties 62 | first 63 | (map (fn [[k v]] [k (first v)])) 64 | (into {})) 65 | parent (->> project :parent first) 66 | dependencies (->> project 67 | :dependencies 68 | first 69 | :dependency 70 | (map (fn [dep] 71 | (->> dep 72 | (map (fn [[k v]] 73 | (let [match (re-find #"^\$\{(.*)\}" 74 | (str (first v)))] 75 | (if match 76 | [k [(-> match 77 | second 78 | keyword 79 | properties-lookup)]] 80 | [k v])))) 81 | (into {})))))] 82 | (if parent 83 | (conj dependencies parent) 84 | dependencies))) 85 | 86 | (defn read-dependency-pipeline [url-set] 87 | "Creates a read depedency pipeline that extracts maven dependecy from a url-set 88 | A url set is looks like [repo '(jar-url pom-url)]" 89 | (let [repo-url (first url-set) 90 | pom-url (-> url-set second first) 91 | c (chan 1 (comp 92 | (map os/parse-xml) 93 | (map #(js->clj % :keywordize-keys true)) 94 | (map extract-deps) 95 | (map clean-deps) 96 | (map #(map mvndep->dep %)) 97 | (map #(into #{} %)) 98 | (map #(conj [] repo-url %)) 99 | ))] 100 | (read-url-chan c pom-url))) 101 | 102 | (defn extract-dependencies [url-set] 103 | (map read-dependency-pipeline url-set)) 104 | 105 | (defn resolve [dep &{:keys [repositories local-repo]} ] 106 | (go-loop [next dep 107 | to-do #{} 108 | done {} 109 | locations #{} 110 | exclusions (:exclusions dep) 111 | status true] 112 | (if next 113 | (do 114 | ;(println repositories) 115 | (let [no-excl (dissoc next :exclusions) 116 | url-set (create-urls-for-dependency repositories no-excl) 117 | urls (map #(-> % second first) url-set) 118 | repo-reqs (extract-dependencies url-set) 119 | tout (timeout 5000) 120 | repo-reqs (conj repo-reqs tout) 121 | [[url deps] ch] (alts! repo-reqs) 122 | to-kill (filter #(not (identical? ch %)) repo-reqs) 123 | real-deps (filter (fn [x] 124 | (empty? (filter #(and 125 | (= (:group %) (:group x)) 126 | (= (:artifact %) (:artifact x))) exclusions))) 127 | deps) 128 | 129 | new-dep (set/union to-do real-deps) 130 | new-locations (set/union locations (conj #{} (assoc no-excl :url url))) 131 | done (into done {no-excl real-deps})] 132 | (if (not (identical? tout ch)) 133 | (do 134 | (map close! to-kill) 135 | (recur (first new-dep) (rest new-dep) done new-locations exclusions true)) 136 | (recur nil nil next [] [] false)))) 137 | [status done locations]))) 138 | 139 | (defn resolve-all [all-deps & opts] 140 | (go 141 | (loop [deps all-deps 142 | dg [] 143 | locations #{} 144 | r-status true] 145 | (if (and r-status (-> deps empty? not)) 146 | (let [to-resolve (first deps) 147 | [status res-dep new-locations] ( % :url is-url-local? ) dep-list) 217 | remote-deps (filter #(-> % :url is-url-local? not) dep-list)] 218 | (if (and offline? (nil? local-repo)) 219 | nil 220 | (if (empty? remote-deps) 221 | (go dep-list) 222 | (retrieve-all local-deps remote-deps local-repo))))) 223 | 224 | 225 | (defn resolve-dependencies 226 | [& {:keys [repositories coordinates retrieve local-repo 227 | proxy mirrors managed-coordinates] 228 | :or {retrieve true}}] 229 | (go 230 | (let [ 231 | l-repo (or local-repo (:local default-repos)) 232 | reps (or repositories (vals (assoc default-repos :local l-repo))) 233 | offline? false 234 | deps-map (map dependency coordinates) 235 | deps-chan (! take! pipeline alts! poll!] :as async] 6 | [clojure.set :as set] 7 | [clojure.string :as str])) 8 | 9 | 10 | (def path (nodejs/require "path")) 11 | (def fs (nodejs/require "fs")) 12 | (def xml2js (nodejs/require "xml2js")) 13 | (def request (nodejs/require "request")) 14 | ;(def dbg (aset request "debug" true)) 15 | 16 | (def HOME-DIR (-> nodejs/process .-env .-HOME)) 17 | (def SEPARATOR (.-sep path)) 18 | 19 | 20 | (defn make-http-request [cout url] 21 | (.get request #js {:url url :encoding nil} 22 | (fn [error response body] 23 | (when (and 24 | (not error) 25 | (= 200 (.-statusCode response))) 26 | ;(println (str "Downloaded from " url)) 27 | ;(println (str "Downloaded from " (count body))) 28 | (put! cout body)))) 29 | cout) 30 | 31 | (defn read-file [cout fpath] 32 | (.readFile fs fpath "utf-8" 33 | (fn [err data] (when-not err 34 | ;(println "Read file " fpath) 35 | (put! cout data)))) 36 | cout) 37 | 38 | (defn create-dir-fully [dir-path] 39 | (if (-> dir-path str/blank? not) 40 | (try 41 | (.mkdirSync fs (str dir-path)) 42 | (catch :default e 43 | (create-dir-fully (.dirname path dir-path)) 44 | (.mkdirSync fs (str dir-path)))))) 45 | 46 | (defn create-conditionally [dir-path] 47 | (try 48 | (.statSync fs dir-path) 49 | (catch :default e 50 | ;TODO catch valid writing errors 51 | (create-dir-fully dir-path)))) 52 | 53 | (defn write-file [file-path content] 54 | (do 55 | (create-conditionally (.dirname path file-path)) 56 | (.writeFileSync fs file-path content) 57 | true)) 58 | 59 | 60 | (defn parse-xml [xmlstring] 61 | "Parses the xml" 62 | (let [x (chan)] 63 | (.parseString xml2js xmlstring #(put! x %2)) 64 | (poll! x))) 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /src/test/clojure/eginez/huckleberry/core_test.cljs: -------------------------------------------------------------------------------- 1 | (ns eginez.huckleberry.core-test 2 | (:require-macros [cljs.core.async.macros :refer [go]]) 3 | (:require [cljs.test :refer-macros [async deftest is testing]] 4 | [cljs.core.async :refer [put! take! chan !] :as async] 5 | [clojure.string :as strg] 6 | [cljs.pprint :as pp] 7 | [eginez.huckleberry.core :as huckleberry])) 8 | 9 | (def test-dep {:group "commons-logging" :artifact "commons-logging" :version "1.1"}) 10 | (def test-dep-exclusions {:group "commons-logging" :artifact "commons-logging" :version "1.1" 11 | :exclusions [{:group "avalon-framework" :artifact "avalon-framework" :version "4.1.3"}]}) 12 | (def test-dep2 {:group "cljs-bach" :artifact "cljs-bach" :version "0.2.0"}) 13 | (def test-dep3 {:group "reagent" :artifact "reagent" :version "0.6.0-alpha2"}) 14 | (def test-dep4 {:group "junit" :artifact "junit" :version "4.12"}) 15 | (def test-dep5 {:group "org.clojure" :artifact "clojure" :version "1.8.0"}) 16 | (def test-dep6 {:group "commons-logging" :artifact "commons-logging" :version "1.1" 17 | :exclusions [{:group "avalon-framework" :artifact "avalon-framework" :version "4.1.3"}]}) 18 | (def test-dep7 {:group "org.clojure" :artifact "core.specs.alpha" :version "0.1.24"}) 19 | (def test-dep8 {:group "org.apache.httpcomponents" :artifact "httpasyncclient" :version "4.1.3"}) 20 | 21 | (def test-url (huckleberry/create-urls-for-dependency (:maven-central huckleberry/default-repos) test-dep)) 22 | 23 | (deftest create-url 24 | (let [urls (huckleberry/create-urls-for-dependency (:local huckleberry/default-repos) test-dep)] 25 | (assert (-> urls first coll? not)))) 26 | 27 | (deftest test-resolve-single 28 | (async done 29 | (go 30 | (let [[status d locations] ( d keys count))) 33 | (assert (= 5 (-> locations count))) 34 | (done))))) 35 | 36 | (deftest test-resolve-single-with-variable-interpolation 37 | (async done 38 | (go 39 | (let [[status d locations] ( d keys count))) 42 | (assert (= 7 (-> locations count))) 43 | (done))))) 44 | 45 | (deftest test-resolve-single-with-parent 46 | (async done 47 | (go 48 | (let [[status d locations] ( d keys count)) 50 | (println (-> locations count)) 51 | (assert (true? status)) 52 | (assert (= 4 (-> d keys count))) 53 | (assert (= 4 (-> locations count))) 54 | (done))))) 55 | 56 | (deftest test-resolve-all-single 57 | (async done 58 | (go 59 | (let [[status d l] ( d second keys count))) 63 | (is (= 5 (-> l count))) 64 | (done))))) 65 | 66 | (deftest test-resolve-all-single-with-exclusion 67 | (async done 68 | (go 69 | (let [[status d l] ( d second keys count))) 73 | (done))))) 74 | 75 | (deftest test-resolve-all-single2 76 | (async done 77 | (go 78 | (let [[status d l] ( d second keys count))) 90 | (print "List of dependencies:") 91 | (pp/pprint l) 92 | (print "Dependency graph:") 93 | (pp/pprint d) 94 | (done) 95 | )))) 96 | 97 | 98 | (deftest test-resolve-dep2 99 | (let [deps '[[commons-logging "1.1"] 100 | [log4j "1.2.15" :exclusions [[javax.mail/mail] 101 | [javax.jms/jms] 102 | com.sun.jdmk/jmxtools 103 | com.sun.jmx/jmxri]]] 104 | ] 105 | (async done 106 | (go 107 | (let [[status dp list] ( dp second keys count))) 112 | (is (= 1 (count (last dp)))) 113 | (done) 114 | ))))) 115 | 116 | (deftest test-retrieve 117 | (async done 118 | (go 119 | (let [dp (path {:group "org.clojure", :artifact "core.cache", :version "0.6.5", :url "tmp/hb/"}))))) 174 | -------------------------------------------------------------------------------- /src/test/clojure/eginez/huckleberry/test_runner.cljs: -------------------------------------------------------------------------------- 1 | (ns eginez.huckleberry.test-runner 2 | (:require [doo.runner :refer-macros [doo-tests]] 3 | [eginez.huckleberry.core-test])) 4 | 5 | (doo-tests 'eginez.huckleberry.core-test) 6 | --------------------------------------------------------------------------------