├── .gitignore ├── LICENSE ├── README.md ├── docs └── scribe.jpg ├── project.clj ├── src ├── s3_journal.clj └── s3_journal │ └── s3.clj └── test ├── credentials.edn.example └── s3_journal_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | /test/credentials.edn 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | .lein-deps-sum 10 | .lein-failures 11 | .lein-plugins 12 | .lein-repl-history 13 | .nrepl* 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of Washington and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](docs/scribe.jpg) 2 | 3 | This library allows an ordered stream of entries to be uploaded to Amazon's S3 datastore. It is implemented using Factual's [durable-queue](https://github.com/factual/durable-queue) library, which means that entries will survive process death, and that memory usage will not be affected by stalls in the uploading process. Despite this, on a `c1.xlarge` AWS instance it can easily journal more than 10k entries/sec, comprising more than 10mb/sec in their compressed serialized form. 4 | 5 | However, this is not a distributed or replicated store, and in the case of node failure may lose data. The amount of data lost will typically be less than 5mb (the minimum upload size allowed by the S3 service), but this library should not be used in any application which cannot tolerate this sort of data loss. The ideal use case is high-throughput logging, especially where external infrastructure is unavailable or impractical. 6 | 7 | ### usage 8 | 9 | ```clj 10 | [factual/s3-journal "0.1.2"] 11 | ``` 12 | 13 | This library exposes only three functions in the `s3-journal` namespace: `journal`, which constructs a journal object that can be written to, `put!`, which writes to the journal, and `stats`, which returns information about the state of the journal. 14 | 15 | All configuration is passed in as a map to `(journal options)`, with the following parameters: 16 | 17 | | name | optional? | description | 18 | |------|-----------|-------------| 19 | | `:s3-access-key` | yes | your AWS access key | 20 | | `:s3-secret-key` | yes | your AWS secret key | 21 | | `:s3-bucket` | yes | the AWS bucket that will be written to, must already exist | 22 | | `:s3-directory-format` | no | the directory format, as a [SimpleDateFormat](http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html) string, should not have leading or trailing slashes, defaults to `yyyy/MM/dd` | 23 | | `:local-directory` | yes | the directory on the local file system that will be used for queueing, will be created if doesn't already exist | 24 | | `:encoder` | no | a function that takes an entry and returns something that can be converted to bytes via [byte-streams](https://github.com/ztellman/byte-streams) | 25 | | `:compressor` | no | Either one of `:gzip`, `:snappy`, `:lzo`, `:bzip2`, or a custom function that takes a sequence of byte-arrays and returns a compressed representation | 26 | | `:delimiter` | no | a delimiter that will be placed between entries, defaults to a newline character | 27 | | `:max-batch-latency` | yes | a value, in milliseconds, of how long entries should be batched before being written to disk | 28 | | `:max-batch-size` | yes | the maximum number of entries that can be batched before being written to disk | 29 | | `:fsync?` | no | describes whether the journal will fsync after writing a batch to disk, defaults to true | 30 | | `:id` | no | a globally unique string describing the journal which is writing to the given location on S3, defaults to the hostname | 31 | | `:expiration` | no | the maximum time, in milliseconds, pending uploads from other processes will be allowed to remain open without being closed by this process. This prevents orphaned multipart uploads from processes which are permanently shutdown persisting forever in a partially updated state (and thus remaining invisible to normal S3 operations). By default this is set to `nil`, which deactivates the expiration behavior. | 32 | | `:shards` | no | the number of top-level directories within the bucket to split the entries across, useful for high-throughput applications, defaults to `nil` | 33 | 34 | Fundamentally, the central tradeoff in these settings are data consistency vs throughput. 35 | 36 | If we persist each entry as it comes in, our throughput is limited to the number of [IOPS](http://en.wikipedia.org/wiki/IOPS) our hardware can handle. However, if we can afford to lose small amounts of data (and we almost certainly can, otherwise we'd be writing each entry to a replicated store individually, rather than in batch), we can bound our loss using the `:max-batch-latency` and `:max-batch-size` parameters. At least one of these parameters must be defined, but usually it's best to define both. Defining our batch size bounds the amount of memory that can be used by the journal, and defining our batch latency bounds the amount of time that a given entry is susceptible to the process dying. Setting `:fsync?` to false can greatly increase throughput, but removes any safety guarantees from the other two parameters - use this parameter only if you're sure you know what you're doing. 37 | 38 | If more than one journal on a given host is writing to the same bucket and directory on S3, a unique identifier for each must be chosen. This identifier should be consistent across process restarts, so that partial uploads from a previous process can be properly handled. One approach is to add a prefix to the hostname, which can be determined by `(s3-journal/hostname)`. 39 | 40 | Calling `(.close journal)` will flush all remaining writes to S3, and only return once they have been successfully written. A journal which has been closed cannot accept any further entries. 41 | 42 | Calling `(stats journal)` returns a data structure in this form: 43 | 44 | ```clj 45 | {:queue {:in-progress 0 46 | :completed 64 47 | :retried 1 48 | :enqueued 64 49 | :num-slabs 1 50 | :num-active-slabs 1} 51 | :enqueued 5000000 52 | :uploaded 5000000} 53 | ``` 54 | 55 | The `:enqueued` key describes how many entries have been enqueued, and the `:uploaded` key how many have been uploaded to S3. The `:queue` values correspond to the statistics reported by the underlying [durable-queue](https://github.com/factual/durable-queue). 56 | 57 | ### logging 58 | 59 | The underlying AWS client libraries will log at the `INFO` level whenever there is an error calling into AWS. This is emulated by `s3-journal` - recoverable errors are logged as `INFO`, and unrecoverable errors, such as corrupted data read back from disk, are logged as WARN. In almost all cases, the journal will continue to work in the face of these errors, but a block of entries may be lost as a result. 60 | 61 | ### license 62 | 63 | Copyright © 2014 Factual, Inc. 64 | 65 | Distributed under the Eclipse Public License version 1.0. 66 | -------------------------------------------------------------------------------- /docs/scribe.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Factual/s3-journal/5a09abd47d9595b3ae8df35c665ad806a9dffd8e/docs/scribe.jpg -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject factual/s3-journal "0.1.3" 2 | :description "Reliable, high-throughput journalling to S3" 3 | :license {:name "Eclipse Public License" 4 | :url "http://www.eclipse.org/legal/epl-v10.html"} 5 | :profiles {:dev {:dependencies [[org.clojure/clojure "1.7.0-alpha6"]]}} 6 | :dependencies [[com.amazonaws/aws-java-sdk-core "1.9.28.1" :exclusions [commons-codec]] 7 | [com.amazonaws/aws-java-sdk-s3 "1.9.28.1" :exclusions [commons-codec]] 8 | [factual/durable-queue "0.1.5"] 9 | [org.clojure/tools.logging "0.3.1"] 10 | [byte-streams "0.2.0"] 11 | [byte-transforms "0.1.4"]] 12 | :jvm-opts ^:replace ["-server" "-Xmx8g"] 13 | :global-vars {*warn-on-reflection* true}) 14 | -------------------------------------------------------------------------------- /src/s3_journal.clj: -------------------------------------------------------------------------------- 1 | (ns s3-journal 2 | (:require 3 | [clojure.tools.logging :as log] 4 | [clojure.repl :as repl] 5 | [byte-streams :as bs] 6 | [byte-transforms :as bt] 7 | [durable-queue :as q] 8 | [clojure.string :as str] 9 | [s3-journal.s3 :as s3] 10 | [clojure.java.io :as io]) 11 | (:import 12 | [java.util.concurrent 13 | LinkedBlockingQueue 14 | Semaphore] 15 | [java.io 16 | Closeable] 17 | [java.util.concurrent.atomic 18 | AtomicLong] 19 | [java.util 20 | ArrayList 21 | Date 22 | TimeZone] 23 | [java.text 24 | SimpleDateFormat] 25 | [java.lang.ref 26 | WeakReference] 27 | [java.lang.management 28 | ManagementFactory] 29 | [com.amazonaws.services.s3.model 30 | PartETag 31 | S3ObjectSummary 32 | PartSummary 33 | ListObjectsRequest 34 | ListPartsRequest 35 | ListMultipartUploadsRequest 36 | MultipartUpload 37 | UploadPartRequest 38 | InitiateMultipartUploadRequest 39 | CompleteMultipartUploadRequest 40 | AbortMultipartUploadRequest] 41 | [com.amazonaws.auth 42 | BasicAWSCredentials] 43 | [com.amazonaws.services.s3 44 | AmazonS3Client] 45 | [com.amazonaws 46 | AmazonServiceException])) 47 | 48 | ;;; 49 | 50 | (defprotocol IExecutor 51 | (put! [_ x] "Enqueues an object for processing, returns true if successful, false if the journal is full.") 52 | (stats [_] "Returns a description of the state of the journal.")) 53 | 54 | (defn- batching-queue [max-size max-time callback] 55 | (assert (or max-size max-time)) 56 | (let [q (LinkedBlockingQueue. (int (or max-size Integer/MAX_VALUE))) 57 | now #(System/currentTimeMillis) 58 | marker (atom (now)) 59 | flush (fn [^LinkedBlockingQueue q] 60 | (let [c (ArrayList.)] 61 | (.drainTo q c) 62 | (locking callback 63 | (if (.isEmpty c) 64 | (callback nil) 65 | (callback c)))))] 66 | 67 | ;; background loop which cleans itself up if the queue 68 | ;; is no longer in use 69 | (when max-time 70 | (let [q (WeakReference. q)] 71 | (future 72 | (loop [] 73 | (when-let [q (.get q)] 74 | (let [m @marker] 75 | (try 76 | (Thread/sleep (max 0 (- (+ m max-time) (now)))) 77 | (when (compare-and-set! marker m (now)) 78 | (flush q)) 79 | (catch Throwable e 80 | (log/warn e "error in batching queue")))) 81 | (recur)))))) 82 | 83 | (reify 84 | IExecutor 85 | (put! [_ x] 86 | (when-not (.offer q x) 87 | (reset! marker (now)) 88 | (flush q) 89 | (recur x))) 90 | java.io.Closeable 91 | (close [_] 92 | (flush q))))) 93 | 94 | ;;; 95 | 96 | (defn- hostname [] 97 | (let [process+host (.getName (ManagementFactory/getRuntimeMXBean))] 98 | (-> process+host 99 | (str/split #"@") 100 | second 101 | (str/replace #"/" "_")))) 102 | 103 | (defn- position->file [id suffix [_ part dir]] 104 | (str dir "/" id "-" (format "%06d" (int (/ part s3/max-parts))) 105 | ".journal" 106 | (when suffix (str "." suffix)))) 107 | 108 | (defn- file->position [path] 109 | (when-let [[dir n] (rest (re-find #"(.*)/.*-(\d+)\.journal" path))] 110 | [0 (* s3/max-parts (Long/parseLong n)) dir])) 111 | 112 | (defn- open-uploads 113 | ([client bucket prefix] 114 | (open-uploads ".*" client bucket prefix)) 115 | ([id client bucket prefix] 116 | (let [re (re-pattern (str id "-(\\d+)\\.journal"))] 117 | (filter 118 | #(re-find re (second %)) 119 | (s3/multipart-uploads client bucket prefix))))) 120 | 121 | (defn- current-file-count 122 | "Returns the number of pre-existing complete and pending uploads in the directory for 123 | the given hostname." 124 | [q id ^AtomicLong enqueued-counter ^Semaphore semaphore client bucket dir] 125 | (let [prefix (str dir "/" id)] 126 | (max 127 | 128 | ;; writes already in AWS 129 | (let [uploads (distinct 130 | (concat 131 | (s3/complete-uploads client bucket prefix) 132 | (map second (s3/multipart-uploads client bucket prefix))))] 133 | (count uploads)) 134 | 135 | ;; pending writes 136 | (let [tasks (q/immediate-task-seq q :s3) 137 | highest-part (->> tasks 138 | (map (fn [task] 139 | (try 140 | (let [[action _ count :as descriptor] @task] 141 | (when (= :conj action) 142 | (when-not (.tryAcquire semaphore count) 143 | (throw 144 | (IllegalStateException. 145 | "Insufficient queue size to handle uploads that are already pending"))) 146 | (.addAndGet enqueued-counter count)) 147 | descriptor) 148 | (catch Throwable e 149 | nil)))) 150 | (map second) 151 | (filter #(= dir (last %))) 152 | (map second) 153 | (apply max 0))] 154 | (doseq [t tasks] 155 | (q/retry! t)) 156 | (long (Math/ceil (/ highest-part s3/max-parts))))))) 157 | 158 | (let [utc (TimeZone/getTimeZone "UTC")] 159 | (defn- format->directory 160 | "Returns the directory location for the current time." 161 | [directory-format] 162 | (.format 163 | (doto (SimpleDateFormat. directory-format) 164 | (.setTimeZone utc)) 165 | (Date.)))) 166 | 167 | ;;; utility functions for inner loop 168 | 169 | (defn- cleanup-expired-uploads 170 | "Identifies any open uploads which are more than `expiration` milliseconds old, and closes them out." 171 | [client bucket date-format expiration] 172 | (let [date-format (doto (SimpleDateFormat. date-format) 173 | (.setTimeZone (TimeZone/getTimeZone "UTC"))) 174 | now (System/currentTimeMillis) 175 | descriptors (->> (open-uploads client bucket nil) 176 | (filter 177 | (fn [[_ ^String path _]] 178 | (when-let [^String file (second (re-matches #".*(/.*?journal.*)" path))] 179 | (let [path' (.substring path 0 (- (count path) (count file)))] 180 | (try 181 | (when (< expiration (- now (.getTime (.parse date-format path')))) 182 | true) 183 | (catch Throwable e 184 | nil))))))) 185 | files (map second descriptors) 186 | parts (map #(s3/parts client %) descriptors)] 187 | 188 | (doseq [[d ps] (map list descriptors parts)] 189 | (try 190 | (s3/end-multipart client ps d) 191 | (catch AmazonServiceException e 192 | (case (.getStatusCode e) 193 | 194 | 404 195 | nil 196 | 197 | 403 198 | (try 199 | (s3/abort-multipart client d) 200 | (catch Throwable e 201 | (log/warn e "error cleaning up old uploads"))) 202 | 203 | (log/warn e "error cleaning up old uploads"))) 204 | (catch Throwable e 205 | (log/warn e "error cleaning up old uploads")))))) 206 | 207 | (defn- advance 208 | "Given a new chunk, returns the location for where it should be appended, and any additional 209 | actions that should be performed." 210 | [[bytes part directory :as pos] directory-format chunk-size] 211 | (let [actions (atom []) 212 | add-actions! #(apply swap! actions conj %&) 213 | directory' (format->directory directory-format) 214 | pos' (if (not= directory directory') 215 | 216 | ;; we've moved to a new directory, so close up previous upload and roll over 217 | (let [pos' [chunk-size 0 directory']] 218 | (add-actions! [:end pos] [:start pos']) 219 | pos') 220 | 221 | (let [part' (if (> bytes s3/min-part-size) 222 | (inc part) 223 | part) 224 | bytes' (if (= part part') 225 | (+ bytes chunk-size) 226 | chunk-size) 227 | pos' [bytes' part' directory]] 228 | 229 | ;; we've hit the maximum part size, so create a new file 230 | (when (and 231 | (not= part part') 232 | (zero? (rem part' s3/max-parts))) 233 | (add-actions! [:end pos] [:start pos'])) 234 | 235 | ;; we've hit the minimum part size threshold, upload the part 236 | (when (> bytes' s3/min-part-size) 237 | (add-actions! [:upload pos'])) 238 | 239 | pos'))] 240 | [pos' @actions])) 241 | 242 | (defn- get-in-state 243 | "A `get-in` function for the upload-state." 244 | [upload-state part dir ks] 245 | (let [file-number (long (/ part s3/max-parts)) 246 | part' (* file-number s3/max-parts)] 247 | (get-in upload-state (cons [part' dir] ks)))) 248 | 249 | (defn- assoc-in-state 250 | "An `assoc-in` function for the upload-state." 251 | [upload-state part dir ks v] 252 | (let [file-number (long (/ part s3/max-parts)) 253 | part' (* file-number s3/max-parts)] 254 | (assoc-in upload-state (cons [part' dir] ks) v))) 255 | 256 | (defn- update-in-state 257 | "An `update-in` function for the upload-state." 258 | [upload-state part dir ks f & args] 259 | (let [file-number (long (/ part s3/max-parts)) 260 | part' (* file-number s3/max-parts)] 261 | (apply update-in upload-state (cons [part' dir] ks) f args))) 262 | 263 | (defn- dissoc-state 264 | "Removes the pending upload described by `[part, dir]`" 265 | [upload-state part dir] 266 | (let [file-number (long (/ part s3/max-parts)) 267 | part' (* file-number s3/max-parts)] 268 | (dissoc upload-state [part' dir]))) 269 | 270 | (defn- upload-descriptor 271 | [upload-state part dir] 272 | (get-in-state upload-state part dir [:descriptor])) 273 | 274 | (defn- initial-upload-state [id client bucket prefix] 275 | (loop [retries 0] 276 | (if-let [vals (try 277 | (let [descriptors (open-uploads id client bucket prefix) 278 | files (map second descriptors)] 279 | (zipmap 280 | (->> files 281 | (map file->position) 282 | (map rest)) 283 | (map 284 | (fn [descriptor] 285 | {:descriptor descriptor 286 | :parts (s3/parts client descriptor)}) 287 | descriptors))) 288 | (catch Throwable e 289 | nil))] 290 | vals 291 | (recur (inc retries))))) 292 | 293 | (defn- upload-part 294 | [client ^AtomicLong upload-counter ^Semaphore semaphore upload-state part dir last?] 295 | (if (get-in-state upload-state part dir [:parts part :uploaded?]) 296 | upload-state 297 | (let [tasks (get-in-state upload-state part dir [:parts part :tasks]) 298 | task-descriptors (map deref tasks) 299 | counts (map #(nth % 2) task-descriptors) 300 | bytes (map #(nth % 3) task-descriptors) 301 | descriptor (upload-descriptor upload-state part dir)] 302 | (try 303 | 304 | (let [rsp (s3/upload-part client 305 | descriptor 306 | (inc (rem part s3/max-parts)) 307 | bytes 308 | last?)] 309 | 310 | (doseq [t tasks] 311 | (q/complete! t)) 312 | 313 | (let [num-entries (reduce + counts)] 314 | (.addAndGet upload-counter num-entries) 315 | (.release semaphore num-entries)) 316 | 317 | (assoc-in-state upload-state part dir [:parts part] rsp)) 318 | 319 | (catch Throwable e 320 | 321 | (log/info e "error uploading part") 322 | 323 | upload-state))))) 324 | 325 | (defn- start-consume-loop 326 | [id ; journal identifier 327 | q ; durable queue 328 | client ; s3 client 329 | bucket ; s3 bucket 330 | prefix ; the unique prefix for this journal (typically only used when sharding) 331 | suffix ; the file suffix (nil by default) 332 | upload-counter ; atomic long for tracking entry uploading 333 | semaphore ; semaphore for controlling maximum pending entries 334 | cleanup ; nil, or a function which is periodically called to clean up dangling uploads 335 | close-latch ; an atom which marks whether the loop should be closed 336 | ] 337 | (let [upload-state (initial-upload-state id client bucket prefix) 338 | last-cleanup (atom 0)] 339 | 340 | (doseq [upload (keys upload-state)] 341 | (q/put! q :s3 [:end (cons 0 upload)])) 342 | 343 | (loop [upload-state upload-state] 344 | 345 | ;; if there's a cleanup function, check if an hour has elapsed since 346 | ;; the last time we called it 347 | (when cleanup 348 | (let [now (System/currentTimeMillis)] 349 | (when (> (- now @last-cleanup) (* 1000 60 60)) 350 | (reset! last-cleanup now) 351 | (cleanup client)))) 352 | 353 | (let [task (try 354 | (if @close-latch 355 | (q/take! q :s3 5000 ::exhausted) 356 | (q/take! q :s3)))] 357 | (when-not (= ::exhausted task) 358 | (let [[action [bytes part dir] & params] (try 359 | @task 360 | (catch Throwable e 361 | ;; something got corrupted, all we 362 | ;; can do is move past it 363 | (log/warn e "error deserializing task") 364 | [:skip])) 365 | descriptor (when part (upload-descriptor upload-state part dir))] 366 | 367 | (recur 368 | (try 369 | (if-not (or (#{:start :flush} action) descriptor) 370 | 371 | ;; the upload this is for no longer is valid, just drop it 372 | (do 373 | (q/complete! task) 374 | upload-state) 375 | 376 | (case action 377 | 378 | :flush 379 | (do 380 | (doseq [[part dir] (keys upload-state)] 381 | (q/put! q :s3 [:end [0 part dir]])) 382 | (q/complete! task) 383 | upload-state) 384 | 385 | ;; new batch of bytes for the part 386 | :conj 387 | (let [[cnt bytes] params] 388 | (if (zero? cnt) 389 | upload-state 390 | (update-in-state upload-state part dir [:parts part :tasks] 391 | #(conj (or % []) task)))) 392 | 393 | ;; actually upload the part 394 | :upload 395 | (let [upload-state' (upload-part client upload-counter semaphore upload-state part dir false)] 396 | (if (get-in-state upload-state' part dir [:parts part :uploaded?]) 397 | (q/complete! task) 398 | (q/retry! task)) 399 | upload-state') 400 | 401 | ;; start a new multipart upload 402 | :start 403 | (let [descriptor (or 404 | (get-in-state upload-state part dir [:descriptor]) 405 | (loop [] 406 | (or 407 | (try 408 | (s3/init-multipart client bucket 409 | (position->file id suffix [0 part dir])) 410 | (catch Throwable e 411 | ;; we can't proceed until this succeeds, so 412 | ;; retrying isn't a valid option 413 | (Thread/sleep 1000) 414 | nil)) 415 | (recur))))] 416 | (q/complete! task) 417 | (assoc-in-state upload-state part dir [:descriptor] descriptor)) 418 | 419 | ;; close out the multipart upload, but only if all the parts have been 420 | ;; successfully uploaded 421 | :end 422 | (let [parts (get-in-state upload-state part dir [:parts]) 423 | non-uploaded (remove #(:uploaded? (val %)) parts) 424 | upload-state' (or 425 | (and 426 | (empty? non-uploaded) 427 | upload-state) 428 | (and 429 | (= 1 (count non-uploaded)) 430 | (let [part' (-> non-uploaded first key)] 431 | (and 432 | (= (rem part' s3/max-parts) (dec (count parts))) 433 | (upload-part client upload-counter semaphore upload-state part' dir true)))) 434 | upload-state) 435 | parts' (get-in-state upload-state' part dir [:parts])] 436 | 437 | (if upload-state' 438 | 439 | ;; we only had one remaining part, check if it was uploaded 440 | (if (->> parts' vals (every? :uploaded?)) 441 | 442 | ;; all the parts have been uploaded, close it out 443 | (do 444 | (s3/end-multipart client 445 | (zipmap 446 | (map #(rem % s3/max-parts) (keys parts')) 447 | (vals parts')) 448 | descriptor) 449 | (q/complete! task) 450 | (dissoc-state upload-state' part dir)) 451 | 452 | (do 453 | (q/retry! task) 454 | (Thread/sleep 1000) 455 | upload-state')) 456 | 457 | ;; wait until we're in a position to close it out 458 | (do 459 | (q/retry! task) 460 | (Thread/sleep 1000) 461 | upload-state))) 462 | 463 | )) 464 | (catch Throwable e 465 | (log/info e "error in task consumption") 466 | (Thread/sleep 1000) 467 | (q/retry! task) 468 | upload-state))))))))) 469 | 470 | ;;; 471 | 472 | (defn- journal- 473 | [{:keys 474 | [s3-access-key 475 | s3-secret-key 476 | s3-bucket 477 | s3-directory-format 478 | local-directory 479 | encoder 480 | compressor 481 | delimiter 482 | fsync? 483 | suffix 484 | max-queue-size 485 | max-batch-latency 486 | max-batch-size 487 | expiration 488 | id] 489 | :or {delimiter "\n" 490 | encoder bs/to-byte-array 491 | id (hostname) 492 | compressor identity 493 | fsync? true 494 | max-queue-size Integer/MAX_VALUE 495 | max-batch-latency (* 1000 60) 496 | s3-directory-format "yyyy/MM/dd"}}] 497 | 498 | (assert local-directory "must define :local-directory for buffering the journal") 499 | 500 | (.mkdirs (io/file local-directory)) 501 | 502 | (let [prefix (second (re-find #"^'(.*)'" s3-directory-format)) 503 | suffix (or suffix 504 | (case compressor 505 | :gzip "gz" 506 | :snappy "snappy" 507 | :bzip2 "bz2" 508 | :lzo "lzo" 509 | nil)) 510 | delimiter (bs/to-byte-array delimiter) 511 | compressor (if (keyword? compressor) 512 | #(bt/compress % compressor) 513 | compressor) 514 | ->bytes (fn [s] 515 | (if (nil? s) 516 | (byte-array 0) 517 | (->> s 518 | (map encoder) 519 | (mapcat #(vector (bs/to-byte-array %) delimiter)) 520 | vec 521 | bs/to-byte-array 522 | compressor 523 | bs/to-byte-array))) 524 | c (s3/client s3-access-key s3-secret-key) 525 | q (q/queues local-directory 526 | {:fsync-put? fsync?}) 527 | initial-directory (format->directory s3-directory-format) 528 | enqueued-counter (AtomicLong. 0) 529 | uploaded-counter (AtomicLong. 0) 530 | pending-semaphore (Semaphore. max-queue-size) 531 | pos (atom 532 | [0 533 | (* s3/max-parts (current-file-count q id enqueued-counter pending-semaphore c s3-bucket initial-directory)) 534 | initial-directory]) 535 | pre-action? #(#{:start} (first %)) 536 | pre-q (batching-queue 537 | max-batch-size 538 | max-batch-latency 539 | (fn [s] 540 | (let [bytes (or (->bytes s) (byte-array 0)) 541 | cnt (count s) 542 | [pos' actions] (advance @pos s3-directory-format (count bytes))] 543 | (.addAndGet enqueued-counter cnt) 544 | (reset! pos pos') 545 | (doseq [a (filter pre-action? actions)] 546 | (q/put! q :s3 a)) 547 | (q/put! q :s3 [:conj pos' cnt bytes]) 548 | (doseq [a (remove pre-action? actions)] 549 | (q/put! q :s3 a))))) 550 | close-latch (atom false)] 551 | 552 | (q/put! q :s3 [:start @pos]) 553 | 554 | (let [consume-loop (future 555 | (try 556 | (start-consume-loop 557 | id 558 | q 559 | c 560 | s3-bucket 561 | prefix 562 | suffix 563 | uploaded-counter 564 | pending-semaphore 565 | (when expiration #(cleanup-expired-uploads % s3-bucket s3-directory-format expiration)) 566 | close-latch) 567 | (catch Throwable e 568 | (log/warn e "error in journal loop"))))] 569 | 570 | ;; consumer loop 571 | (reify IExecutor 572 | (stats [_] 573 | (let [uploaded (.get uploaded-counter) 574 | enqueued (.get enqueued-counter)] 575 | {:enqueued enqueued 576 | :uploaded uploaded 577 | :queue (get (q/stats q) "s3")})) 578 | (put! [_ x] 579 | (if @close-latch 580 | (throw (IllegalStateException. "attempting to write to a closed journal")) 581 | (boolean 582 | (and 583 | (.tryAcquire pending-semaphore) 584 | (put! pre-q x))))) 585 | Closeable 586 | (close [_] 587 | (.close ^java.io.Closeable pre-q) 588 | (q/put! q :s3 [:flush]) 589 | (reset! close-latch true) 590 | @consume-loop 591 | nil))))) 592 | 593 | (def ^:private shard-ids 594 | (concat 595 | (range 10) 596 | (map char (range (int \a) (inc (int \z)))))) 597 | 598 | (defn journal 599 | [{:keys 600 | [s3-access-key 601 | s3-secret-key 602 | s3-bucket 603 | s3-directory-format 604 | local-directory 605 | encoder 606 | compressor 607 | delimiter 608 | fsync? 609 | max-batch-latency 610 | max-batch-size 611 | expiration 612 | id 613 | max-queue-size 614 | suffix 615 | shards] 616 | :or {delimiter "\n" 617 | encoder bs/to-byte-array 618 | id (hostname) 619 | compressor identity 620 | fsync? true 621 | max-batch-latency (* 1000 60) 622 | max-queue-size Integer/MAX_VALUE 623 | s3-directory-format "yyyy/MM/dd"} 624 | :as options}] 625 | "Creates a journal that will write to S3." 626 | (if shards 627 | 628 | ;; we want to shard the streams 629 | (do 630 | (assert (<= shards 36)) 631 | (let [journals (zipmap 632 | (range shards) 633 | (map 634 | (fn [shard] 635 | (journal- 636 | (-> options 637 | (update-in [:max-queue-size] 638 | #(if % (/ % shards) Integer/MAX_VALUE)) 639 | (update-in [:s3-directory-format] 640 | #(str \' (nth shard-ids shard) "'/" %)) 641 | (update-in [:local-directory] 642 | #(when % (str local-directory "/" (nth shard-ids shard))))))) 643 | (range shards))) 644 | counter (AtomicLong. 0)] 645 | (reify IExecutor 646 | (stats [_] 647 | (let [stats (->> journals vals (map stats))] 648 | (merge 649 | (->> stats (map #(dissoc % :queue)) (apply merge-with +)) 650 | {:queue (->> stats (map :queue) (apply merge-with +))}))) 651 | (put! [_ x] 652 | (put! 653 | (journals (rem (.getAndIncrement counter) shards)) 654 | x)) 655 | java.io.Closeable 656 | (close [_] 657 | (doseq [^Closeable j (vals journals)] 658 | (.close j)))))) 659 | 660 | (journal- options))) 661 | -------------------------------------------------------------------------------- /src/s3_journal/s3.clj: -------------------------------------------------------------------------------- 1 | (ns s3-journal.s3 2 | (:require 3 | [byte-streams :as bs] 4 | [clojure.string :as str] 5 | [clojure.tools.logging :as log] 6 | [clojure.data :as data]) 7 | (:import 8 | [com.amazonaws.regions 9 | Region 10 | Regions] 11 | [com.amazonaws.services.s3.model 12 | PartETag 13 | S3ObjectSummary 14 | PartSummary 15 | ListObjectsRequest 16 | ListPartsRequest 17 | ListMultipartUploadsRequest 18 | MultipartUpload 19 | UploadPartRequest 20 | UploadPartResult 21 | InitiateMultipartUploadRequest 22 | CompleteMultipartUploadRequest 23 | AbortMultipartUploadRequest] 24 | [com.amazonaws.auth 25 | BasicAWSCredentials] 26 | [com.amazonaws 27 | AmazonServiceException] 28 | [com.amazonaws.services.s3 29 | AmazonS3Client])) 30 | 31 | ;;; 32 | 33 | (def min-part-size (* 5 1024 1024)) 34 | 35 | (def max-parts 500) 36 | 37 | ;;; 38 | 39 | (defn ^AmazonS3Client client 40 | "Returns an S3 client that can be used with the other functions." 41 | [access-key secret-key] 42 | (AmazonS3Client. 43 | (BasicAWSCredentials. 44 | access-key 45 | secret-key))) 46 | 47 | (defn complete-uploads 48 | "Returns a list of strings representing complete uploads." 49 | [^AmazonS3Client client bucket prefix] 50 | (let [rsp (.listObjects client 51 | (doto (ListObjectsRequest.) 52 | (.setBucketName bucket) 53 | (.setPrefix prefix) 54 | (.setMaxKeys (int 1e5))))] 55 | (->> rsp 56 | .getObjectSummaries 57 | (map (fn [^S3ObjectSummary s] 58 | (.getKey s)))))) 59 | 60 | (defn multipart-uploads 61 | "Returns a list of tuples representing open multipart uploads." 62 | [^AmazonS3Client client bucket prefix] 63 | (let [rsp (.listMultipartUploads client 64 | (doto (ListMultipartUploadsRequest. bucket) 65 | (.setPrefix prefix) 66 | (.setMaxUploads (int 1e5))))] 67 | (->> rsp 68 | .getMultipartUploads 69 | (map (fn [^MultipartUpload u] 70 | [bucket (.getKey u) (.getUploadId u)]))))) 71 | 72 | (defn parts 73 | "Given a multipart upload descriptor, returns a map of part numbers onto etags." 74 | [^AmazonS3Client client [bucket key upload-id :as upload]] 75 | (try 76 | (let [rsp (.listParts client 77 | (doto (ListPartsRequest. bucket key upload-id) 78 | (.setMaxParts (int 1e5))))] 79 | (->> rsp 80 | .getParts 81 | (map (fn [^PartSummary s] 82 | [(dec (.getPartNumber s)) 83 | {:tag (.getETag s) 84 | :size (.getSize s) 85 | :uploaded? true}])) 86 | (into {}))) 87 | (catch AmazonServiceException e 88 | (case (.getStatusCode e) 89 | 90 | 404 91 | nil 92 | 93 | (throw e))))) 94 | 95 | (defn init-multipart 96 | "Creates a new multipart upload at the given `key`." 97 | [^AmazonS3Client client bucket key] 98 | (let [rsp (.initiateMultipartUpload client 99 | (InitiateMultipartUploadRequest. bucket key))] 100 | [bucket key (.getUploadId rsp)])) 101 | 102 | (defn upload-part 103 | "Uploads a part. If an upload with the `part-number` already exists, this is a no-op." 104 | [^AmazonS3Client client [bucket key upload-id] part-number contents last?] 105 | (let [ary (bs/to-byte-array contents) 106 | _ (assert (or last? (> (count ary) min-part-size))) 107 | rsp (try 108 | (.uploadPart client 109 | (doto (UploadPartRequest.) 110 | (.setBucketName bucket) 111 | (.setKey key) 112 | (.setUploadId upload-id) 113 | (.setPartNumber part-number) 114 | (.setPartSize (count ary)) 115 | (.setInputStream (bs/to-input-stream (or ary (byte-array 0)))))) 116 | (catch AmazonServiceException e 117 | (case (.getStatusCode e) 118 | 119 | 404 120 | nil 121 | 122 | (throw e))))] 123 | {:tag (when rsp (.getETag ^UploadPartResult rsp)) 124 | :uploaded? true 125 | :size (count ary) 126 | :last? last?})) 127 | 128 | (defn abort-multipart 129 | "Cancels an open multipart upload." 130 | [^AmazonS3Client client [bucket key upload-id]] 131 | (.abortMultipartUpload client 132 | (AbortMultipartUploadRequest. bucket key upload-id))) 133 | 134 | (defn end-multipart 135 | "Completes a multipart upload." 136 | [^AmazonS3Client client part->descriptor [bucket key upload-id :as upload]] 137 | (try 138 | (if (empty? part->descriptor) 139 | (abort-multipart client upload) 140 | (.completeMultipartUpload client 141 | (CompleteMultipartUploadRequest. bucket key upload-id 142 | (->> part->descriptor 143 | (sort-by first) 144 | (map 145 | (fn [[n {:keys [tag]}]] 146 | (PartETag. (inc n) tag))))))) 147 | (catch AmazonServiceException e 148 | (case (.getStatusCode e) 149 | 150 | #_400 151 | #_(abort-multipart client upload) 152 | 153 | 404 154 | nil 155 | 156 | (throw e))) 157 | (catch Throwable e 158 | (throw e)))) 159 | -------------------------------------------------------------------------------- /test/credentials.edn.example: -------------------------------------------------------------------------------- 1 | {:access-key "ABC" 2 | :secret-key "abc123" 3 | :bucket "a-bucket"} 4 | -------------------------------------------------------------------------------- /test/s3_journal_test.clj: -------------------------------------------------------------------------------- 1 | (ns s3-journal-test 2 | (:require 3 | [clojure.java.shell :as sh] 4 | [byte-streams :as bs] 5 | [clojure.test :refer :all] 6 | [s3-journal :as s] 7 | [s3-journal.s3 :as s3]) 8 | (:import 9 | [com.amazonaws.services.s3.model 10 | GetObjectRequest 11 | S3Object] 12 | [com.amazonaws.services.s3 13 | AmazonS3Client])) 14 | 15 | (defn sometimes-explode [f] 16 | (fn [& args] 17 | (assert (not= 1 (rand-int 3)) "rolled the dice, you lost") 18 | (apply f args))) 19 | 20 | (defn explode-in-streaks [f] 21 | (fn [& args] 22 | (assert 23 | (if-not (not= 1 (rem (long (/ (System/currentTimeMillis) 1e4)) 10)) 24 | (do (Thread/sleep 1000) false) 25 | true) 26 | "things are down for a bit") 27 | (apply f args))) 28 | 29 | (defn get-test-object [^AmazonS3Client client bucket directory] 30 | (let [test-objects (s3/complete-uploads client bucket directory)] 31 | (apply concat 32 | (map 33 | (fn [object] 34 | (->> object 35 | (GetObjectRequest. bucket) 36 | (.getObject client) 37 | .getObjectContent 38 | bs/to-reader 39 | line-seq 40 | (map #(Long/parseLong %)) 41 | doall)) 42 | (sort test-objects))))) 43 | 44 | (defn clear-test-folder [^AmazonS3Client client bucket directory] 45 | (doseq [u (s3/multipart-uploads client bucket directory)] 46 | (s3/abort-multipart client u)) 47 | (doseq [o (s3/complete-uploads client bucket directory)] 48 | (.deleteObject client bucket o))) 49 | 50 | (defmacro with-random-errors [& body] 51 | `(with-redefs [s3/init-multipart (sometimes-explode s3/init-multipart) 52 | s3/upload-part (sometimes-explode s3/upload-part) 53 | s3/end-multipart (sometimes-explode s3/end-multipart) 54 | ] 55 | ~@body)) 56 | 57 | (defmacro with-intermittent-errors [& body] 58 | `(with-redefs [s3/init-multipart (explode-in-streaks s3/init-multipart) 59 | s3/upload-part (explode-in-streaks s3/upload-part) 60 | s3/end-multipart (explode-in-streaks s3/end-multipart) 61 | ] 62 | ~@body)) 63 | 64 | (defn run-stress-test [access-key secret-key bucket] 65 | (with-redefs [s3/max-parts 4] 66 | (let [s3-dir "stress-test" 67 | directory "/tmp/journal-stress-test" 68 | c (s3/client access-key secret-key) 69 | _ (clear-test-folder c bucket (str "0/" s3-dir)) 70 | _ (sh/sh "rm" "-rf" directory) 71 | j (s/journal 72 | {:shards 1 73 | :s3-access-key access-key 74 | :s3-secret-key secret-key 75 | :s3-bucket bucket 76 | :s3-directory-format (str \' s3-dir \' "/yy/MM/dd/hh/mm") 77 | :local-directory directory 78 | :max-batch-size 1e5}) 79 | n 5e6] 80 | (dotimes [i n] 81 | (s/put! j (str (inc i)))) 82 | (prn (s/stats j)) 83 | (.close j) 84 | (prn (s/stats j)) 85 | (= 86 | (get-test-object c "journal-test" (str "0/" s3-dir)) 87 | (map inc (range n)))))) 88 | 89 | (deftest test-journalling 90 | (let [{:keys [access-key secret-key bucket]} (read-string (slurp "test/credentials.edn"))] 91 | (is 92 | (and 93 | (run-stress-test access-key secret-key bucket) 94 | (with-random-errors (run-stress-test access-key secret-key bucket)) 95 | (with-intermittent-errors (run-stress-test access-key secret-key bucket)))))) 96 | --------------------------------------------------------------------------------