├── .gitignore ├── boot.properties ├── src └── hashobject │ ├── boot_s3 │ ├── merge.clj │ ├── fs.clj │ ├── s3.clj │ └── core.clj │ └── boot_s3.clj ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | -------------------------------------------------------------------------------- /boot.properties: -------------------------------------------------------------------------------- 1 | #https://github.com/boot-clj/boot 2 | #Sat Oct 03 13:45:25 PDT 2015 3 | BOOT_CLOJURE_VERSION=1.7.0 4 | BOOT_VERSION=2.7.2 5 | -------------------------------------------------------------------------------- /src/hashobject/boot_s3/merge.clj: -------------------------------------------------------------------------------- 1 | (ns hashobject.boot-s3.merge 2 | (:require [clojure.set :as s])) 3 | 4 | (defn generate-deltas [local-file-details s3-file-details] 5 | (remove #(contains? s3-file-details 6 | (select-keys % [:path :md5])) 7 | local-file-details)) 8 | -------------------------------------------------------------------------------- /src/hashobject/boot_s3/fs.clj: -------------------------------------------------------------------------------- 1 | (ns hashobject.boot-s3.fs 2 | (:require [boot.core :as boot] 3 | [pandect.core :as p]) 4 | (:import [java.io File] 5 | [java.util.regex Pattern])) 6 | 7 | (declare relative-path) 8 | (declare path->file-details) 9 | 10 | (defn analyse-local-files 11 | "Analyse a local directory returnings a set 12 | of file details describing the relative path 13 | and md5 checksum of all the files (recursively) 14 | under the directory." 15 | [dir-path files] 16 | (->> files 17 | (map (partial path->file-details dir-path)) 18 | set)) 19 | 20 | ;; Private Helper Functions 21 | 22 | (defn- root-path-regex [root] 23 | (let [updated-root (.replace root File/separator "/")] 24 | (str "^" (str updated-root "/")))) 25 | 26 | (defn- relative-path [root target] 27 | (-> target 28 | (.replace File/separator "/") 29 | (.replaceAll (root-path-regex root) ""))) 30 | 31 | (defn- path->file-details [root-path tmp-file] 32 | (let [file-path (:path tmp-file) 33 | rel-path (relative-path root-path file-path) 34 | md5 (p/md5 (boot/tmp-file tmp-file))] 35 | {:path rel-path :md5 md5 :tmp-file tmp-file})) 36 | -------------------------------------------------------------------------------- /src/hashobject/boot_s3/s3.clj: -------------------------------------------------------------------------------- 1 | (ns hashobject.boot-s3.s3 2 | (:require [aws.sdk.s3 :as s3])) 3 | 4 | (defn get-file-details-for 5 | "Get the file details for the file in s3. 6 | Returns nil if there is no file at the given key." 7 | [cred bucket-name key] 8 | (try 9 | (let [response (s3/get-object-metadata cred bucket-name key)] 10 | (assoc response :key key)) 11 | (catch com.amazonaws.services.s3.model.AmazonS3Exception e 12 | (when-not (= 404 (.getStatusCode e)) 13 | (throw e))))) 14 | 15 | (defn- response->file-details [response] 16 | {:path (:key response) :md5 (:etag response)} ) 17 | 18 | (defn analyse-s3-bucket [cred bucket-name file-paths] 19 | (let [s3-lookup (partial get-file-details-for cred bucket-name) 20 | bucket-sync-state {:bucket-name bucket-name 21 | :remote-file-details []}] 22 | (if-not (s3/bucket-exists? cred bucket-name) 23 | (assoc bucket-sync-state :errors [(str "No bucket " bucket-name)]) 24 | (->> file-paths 25 | (map s3-lookup) 26 | (map response->file-details) 27 | (remove nil?) 28 | (set))))) 29 | 30 | (defn put-file [cred bucket-name key file metadata permissions] 31 | (let [grants (map #(apply s3/grant %) permissions)] 32 | (apply s3/put-object cred bucket-name key file metadata grants))) 33 | -------------------------------------------------------------------------------- /src/hashobject/boot_s3.clj: -------------------------------------------------------------------------------- 1 | (ns hashobject.boot-s3 2 | {:boot/export-tasks true} 3 | (:require [boot.core :as boot] 4 | [boot.util :as u] 5 | [hashobject.boot-s3.core :as s3])) 6 | 7 | 8 | (def ^:private 9 | +defaults+ {:source "public" 10 | :permissions [[:all-users :read]]}) 11 | 12 | (boot/deftask s3-sync 13 | "Sync local directory to AWS S3" 14 | [s source PATH str "subdirectory in :target-path to upload to s3" 15 | b bucket BUCKET str "s3 bucket name" 16 | a access-key ACCESS_KEY str "s3 access key" 17 | k secret-key SECRET str "s3 secret key" 18 | m metadata META {kw str} "a map with metadata to set on the objects, passed through to clj-aws-s3" 19 | p permissions PERMS [[kw kw]] "a seq of 2-tuples of `[grantee permission]`, passed through to clj-aws-s3" 20 | f force bool "Set to `true` to force upload of all objects"] 21 | (let [options (merge +defaults+ *opts*) 22 | cred (select-keys options [:access-key :secret-key]) 23 | s3-opts {:metadata metadata :permissions (:permissions options)}] 24 | (boot/with-post-wrap fileset 25 | (let [files (->> (boot/output-files fileset) 26 | (boot/by-re [(re-pattern (str "^" (:source options)))]))] 27 | (u/info "Start upload to AWS S3.\n") 28 | (s3/sync-to-s3 cred files (:source options) (:bucket options) s3-opts force) 29 | (u/info "Uploaded to AWS S3.\n"))))) 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # boot-s3 2 | 3 | [![Dependencies Status](https://jarkeeper.com/hashobject/boot-s3/status.svg)](https://jarkeeper.com/hashobject/boot-s3) 4 | [![Downloads](https://jarkeeper.com/hashobject/boot-s3/downloads.svg)](https://jarkeeper.com/hashobject/boot-s3) 5 | 6 | [Boot](http://boot-clj.com/) task to sync local directory with AWS S3. 7 | Most of the code borrowed from [lein-s3-sync](https://github.com/kanej/lein-s3-sync). 8 | This task works in the efficient way and uploads only new or changed files to the s3. 9 | 10 | ## Install 11 | 12 | ``` 13 | [hashobject/boot-s3 "0.1.3-SNAPSHOT"] 14 | ``` 15 | 16 | ## Usage 17 | 18 | Add library 19 | ``` 20 | (require '[hashobject.boot-s3 :refer :all]) 21 | ``` 22 | and use it: 23 | 24 | ``` 25 | boot s3-sync -h 26 | Sync local directory to AWS S3 27 | 28 | Options: 29 | -h, --help Print this help info. 30 | -s, --source PATH PATH sets subdirectory in :target-path to upload to s3. 31 | -b, --bucket BUCKET BUCKET sets s3 bucket name. 32 | -a, --access-key ACCESS_KEY ACCESS_KEY sets s3 access key. 33 | -k, --secret-key SECRET SECRET sets s3 secret key. 34 | -m, --metadata META Conj META onto a map with metadata to set on the objects, passed through to clj-aws-s3 35 | -p, --permissions PERMS Conj PERMS onto a seq of 2-tuples of `[grantee permission]`, passed through to clj-aws-s3 36 | -f, --force Set to `true` to force upload of all objects 37 | ``` 38 | 39 | ## Contributions 40 | 41 | We love contributions. Please submit your pull requests. 42 | 43 | 44 | ## License 45 | 46 | Copyright © 2013-2017 Hashobject Ltd (team@hashobject.com). 47 | 48 | Distributed under the [Eclipse Public License](http://opensource.org/licenses/eclipse-1.0). 49 | -------------------------------------------------------------------------------- /src/hashobject/boot_s3/core.clj: -------------------------------------------------------------------------------- 1 | (ns hashobject.boot-s3.core 2 | (:require [boot.core :as boot] 3 | [hashobject.boot-s3.fs :as fs] 4 | [hashobject.boot-s3.s3 :as s3] 5 | [hashobject.boot-s3.merge :as m])) 6 | 7 | (declare capture-file-details) 8 | (declare calculate-deltas) 9 | (declare push-deltas-to-s3) 10 | 11 | (declare print-delta-summary) 12 | (declare print-sync-complete-message) 13 | 14 | (defn sync-to-s3 15 | "Syncronise the local directory 'dir-path' to the S3 bucket 'bucket-name'." 16 | ([aws-credentials files dir-path bucket-name] 17 | (sync-to-s3 aws-credentials dir-path bucket-name {} false)) 18 | ([aws-credentials files dir-path bucket-name options] 19 | (sync-to-s3 aws-credentials dir-path bucket-name options false)) 20 | ([aws-credentials files dir-path bucket-name options force] 21 | (let [sync-state* {:aws-credentials aws-credentials 22 | :files files 23 | :dir-path dir-path 24 | :bucket-name bucket-name 25 | :options options} 26 | sync-state (if force 27 | (assoc sync-state* :deltas (fs/analyse-local-files dir-path files)) 28 | (-> sync-state* 29 | capture-file-details 30 | calculate-deltas))] 31 | (-> sync-state 32 | print-delta-summary 33 | push-deltas-to-s3 34 | print-sync-complete-message)))) 35 | 36 | ;; Private functions 37 | 38 | (def padding (apply str (take 30 (repeat " ")))) 39 | 40 | (defn- capture-file-details 41 | "Pull the local directories file details and the S3 buckets file details 42 | and associate them with the sync-state." 43 | [{:keys [aws-credentials dir-path files bucket-name] :as sync-state}] 44 | (let [local-file-details (fs/analyse-local-files dir-path files) 45 | file-paths (map :path local-file-details) 46 | remote-file-details (s3/analyse-s3-bucket aws-credentials bucket-name file-paths)] 47 | (merge sync-state {:local-file-details local-file-details 48 | :remote-file-details remote-file-details}))) 49 | 50 | (defn- calculate-deltas 51 | "Based on the local file details and the remote file details, calculate 52 | which local files need to be pushed and which do not." 53 | [{:keys [errors local-file-details remote-file-details] :as sync-state}] 54 | (if (empty? errors) 55 | (let [deltas (m/generate-deltas local-file-details remote-file-details)] 56 | (assoc sync-state :deltas deltas)))) 57 | 58 | (defn- push-deltas-to-s3 59 | "Pushes the local files named in the delta list to S3." 60 | [{:keys [errors aws-credentials bucket-name deltas options] :as sync-state}] 61 | (when (empty? errors) 62 | (loop [deltas deltas] 63 | (if (not (empty? deltas)) 64 | (let [{:keys [path tmp-file]} (first deltas) 65 | {:keys [metadata permissions]} (merge-with merge 66 | options 67 | (:hashobject/boot-s3 tmp-file))] 68 | (print " " path "uploading ...") 69 | 70 | (s3/put-file 71 | aws-credentials 72 | bucket-name 73 | path 74 | (boot/tmp-file tmp-file) 75 | metadata 76 | permissions) 77 | 78 | (println "\r " path "done." padding) 79 | (recur (rest deltas)))))) 80 | sync-state) 81 | 82 | ;; Print Functions 83 | 84 | (defn- print-delta-summary [{:keys [errors deltas] :as sync-state}] 85 | (cond 86 | (not (empty? errors)) nil 87 | (empty? deltas) (println "\rThere are no local changes to push." padding) 88 | (= 1 (count deltas)) (println "\nThere is 1 local file change to upload:") 89 | :default (println "\nThere are" (count deltas) "local file changes to upload:")) 90 | sync-state) 91 | 92 | (defn- print-sync-complete-message [{:keys [errors deltas]}] 93 | (cond 94 | (not (empty? errors)) (println (str "\r" (first errors) padding)) 95 | (not (empty? deltas)) (println "Sync complete."))) 96 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Eclipse Public License - v 1.0 2 | 3 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 4 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 5 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 6 | 7 | 1. DEFINITIONS 8 | 9 | "Contribution" means: 10 | 11 | a) in the case of the initial Contributor, the initial code and documentation 12 | distributed under this Agreement, and 13 | b) in the case of each subsequent Contributor: 14 | i) changes to the Program, and 15 | ii) additions to the Program; 16 | 17 | where such changes and/or additions to the Program originate from and are 18 | distributed by that particular Contributor. A Contribution 'originates' 19 | from a Contributor if it was added to the Program by such Contributor 20 | itself or anyone acting on such Contributor's behalf. Contributions do not 21 | include additions to the Program which: (i) are separate modules of 22 | software distributed in conjunction with the Program under their own 23 | license agreement, and (ii) are not derivative works of the Program. 24 | 25 | "Contributor" means any person or entity that distributes the Program. 26 | 27 | "Licensed Patents" mean patent claims licensable by a Contributor which are 28 | necessarily infringed by the use or sale of its Contribution alone or when 29 | combined with the Program. 30 | 31 | "Program" means the Contributions distributed in accordance with this 32 | Agreement. 33 | 34 | "Recipient" means anyone who receives the Program under this Agreement, 35 | including all Contributors. 36 | 37 | 2. GRANT OF RIGHTS 38 | a) Subject to the terms of this Agreement, each Contributor hereby grants 39 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 40 | reproduce, prepare derivative works of, publicly display, publicly 41 | perform, distribute and sublicense the Contribution of such Contributor, 42 | if any, and such derivative works, in source code and object code form. 43 | b) Subject to the terms of this Agreement, each Contributor hereby grants 44 | Recipient a non-exclusive, worldwide, royalty-free patent license under 45 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 46 | transfer the Contribution of such Contributor, if any, in source code and 47 | object code form. This patent license shall apply to the combination of 48 | the Contribution and the Program if, at the time the Contribution is 49 | added by the Contributor, such addition of the Contribution causes such 50 | combination to be covered by the Licensed Patents. The patent license 51 | shall not apply to any other combinations which include the Contribution. 52 | No hardware per se is licensed hereunder. 53 | c) Recipient understands that although each Contributor grants the licenses 54 | to its Contributions set forth herein, no assurances are provided by any 55 | Contributor that the Program does not infringe the patent or other 56 | intellectual property rights of any other entity. Each Contributor 57 | disclaims any liability to Recipient for claims brought by any other 58 | entity based on infringement of intellectual property rights or 59 | otherwise. As a condition to exercising the rights and licenses granted 60 | hereunder, each Recipient hereby assumes sole responsibility to secure 61 | any other intellectual property rights needed, if any. For example, if a 62 | third party patent license is required to allow Recipient to distribute 63 | the Program, it is Recipient's responsibility to acquire that license 64 | before distributing the Program. 65 | d) Each Contributor represents that to its knowledge it has sufficient 66 | copyright rights in its Contribution, if any, to grant the copyright 67 | license set forth in this Agreement. 68 | 69 | 3. REQUIREMENTS 70 | 71 | A Contributor may choose to distribute the Program in object code form under 72 | its own license agreement, provided that: 73 | 74 | a) it complies with the terms and conditions of this Agreement; and 75 | b) its license agreement: 76 | i) effectively disclaims on behalf of all Contributors all warranties 77 | and conditions, express and implied, including warranties or 78 | conditions of title and non-infringement, and implied warranties or 79 | conditions of merchantability and fitness for a particular purpose; 80 | ii) effectively excludes on behalf of all Contributors all liability for 81 | damages, including direct, indirect, special, incidental and 82 | consequential damages, such as lost profits; 83 | iii) states that any provisions which differ from this Agreement are 84 | offered by that Contributor alone and not by any other party; and 85 | iv) states that source code for the Program is available from such 86 | Contributor, and informs licensees how to obtain it in a reasonable 87 | manner on or through a medium customarily used for software exchange. 88 | 89 | When the Program is made available in source code form: 90 | 91 | a) it must be made available under this Agreement; and 92 | b) a copy of this Agreement must be included with each copy of the Program. 93 | Contributors may not remove or alter any copyright notices contained 94 | within the Program. 95 | 96 | Each Contributor must identify itself as the originator of its Contribution, 97 | if 98 | any, in a manner that reasonably allows subsequent Recipients to identify the 99 | originator of the Contribution. 100 | 101 | 4. COMMERCIAL DISTRIBUTION 102 | 103 | Commercial distributors of software may accept certain responsibilities with 104 | respect to end users, business partners and the like. While this license is 105 | intended to facilitate the commercial use of the Program, the Contributor who 106 | includes the Program in a commercial product offering should do so in a manner 107 | which does not create potential liability for other Contributors. Therefore, 108 | if a Contributor includes the Program in a commercial product offering, such 109 | Contributor ("Commercial Contributor") hereby agrees to defend and indemnify 110 | every other Contributor ("Indemnified Contributor") against any losses, 111 | damages and costs (collectively "Losses") arising from claims, lawsuits and 112 | other legal actions brought by a third party against the Indemnified 113 | Contributor to the extent caused by the acts or omissions of such Commercial 114 | Contributor in connection with its distribution of the Program in a commercial 115 | product offering. The obligations in this section do not apply to any claims 116 | or Losses relating to any actual or alleged intellectual property 117 | infringement. In order to qualify, an Indemnified Contributor must: 118 | a) promptly notify the Commercial Contributor in writing of such claim, and 119 | b) allow the Commercial Contributor to control, and cooperate with the 120 | Commercial Contributor in, the defense and any related settlement 121 | negotiations. The Indemnified Contributor may participate in any such claim at 122 | its own expense. 123 | 124 | For example, a Contributor might include the Program in a commercial product 125 | offering, Product X. That Contributor is then a Commercial Contributor. If 126 | that Commercial Contributor then makes performance claims, or offers 127 | warranties related to Product X, those performance claims and warranties are 128 | such Commercial Contributor's responsibility alone. Under this section, the 129 | Commercial Contributor would have to defend claims against the other 130 | Contributors related to those performance claims and warranties, and if a 131 | court requires any other Contributor to pay any damages as a result, the 132 | Commercial Contributor must pay those damages. 133 | 134 | 5. NO WARRANTY 135 | 136 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN 137 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 138 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each 140 | Recipient is solely responsible for determining the appropriateness of using 141 | and distributing the Program and assumes all risks associated with its 142 | exercise of rights under this Agreement , including but not limited to the 143 | risks and costs of program errors, compliance with applicable laws, damage to 144 | or loss of data, programs or equipment, and unavailability or interruption of 145 | operations. 146 | 147 | 6. DISCLAIMER OF LIABILITY 148 | 149 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 150 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 151 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 152 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 153 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 154 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 155 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 156 | OF SUCH DAMAGES. 157 | 158 | 7. GENERAL 159 | 160 | If any provision of this Agreement is invalid or unenforceable under 161 | applicable law, it shall not affect the validity or enforceability of the 162 | remainder of the terms of this Agreement, and without further action by the 163 | parties hereto, such provision shall be reformed to the minimum extent 164 | necessary to make such provision valid and enforceable. 165 | 166 | If Recipient institutes patent litigation against any entity (including a 167 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 168 | (excluding combinations of the Program with other software or hardware) 169 | infringes such Recipient's patent(s), then such Recipient's rights granted 170 | under Section 2(b) shall terminate as of the date such litigation is filed. 171 | 172 | All Recipient's rights under this Agreement shall terminate if it fails to 173 | comply with any of the material terms or conditions of this Agreement and does 174 | not cure such failure in a reasonable period of time after becoming aware of 175 | such noncompliance. If all Recipient's rights under this Agreement terminate, 176 | Recipient agrees to cease use and distribution of the Program as soon as 177 | reasonably practicable. However, Recipient's obligations under this Agreement 178 | and any licenses granted by Recipient relating to the Program shall continue 179 | and survive. 180 | 181 | Everyone is permitted to copy and distribute copies of this Agreement, but in 182 | order to avoid inconsistency the Agreement is copyrighted and may only be 183 | modified in the following manner. The Agreement Steward reserves the right to 184 | publish new versions (including revisions) of this Agreement from time to 185 | time. No one other than the Agreement Steward has the right to modify this 186 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 187 | Eclipse Foundation may assign the responsibility to serve as the Agreement 188 | Steward to a suitable separate entity. Each new version of the Agreement will 189 | be given a distinguishing version number. The Program (including 190 | Contributions) may always be distributed subject to the version of the 191 | Agreement under which it was received. In addition, after a new version of the 192 | Agreement is published, Contributor may elect to distribute the Program 193 | (including its Contributions) under the new version. Except as expressly 194 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 195 | licenses to the intellectual property of any Contributor under this Agreement, 196 | whether expressly, by implication, estoppel or otherwise. All rights in the 197 | Program not expressly granted under this Agreement are reserved. 198 | 199 | This Agreement is governed by the laws of the State of New York and the 200 | intellectual property laws of the United States of America. No party to this 201 | Agreement will bring a legal action under this Agreement more than one year 202 | after the cause of action arose. Each party waives its rights to a jury trial in 203 | any resulting litigation. 204 | 205 | --------------------------------------------------------------------------------