├── .github └── workflows │ └── main.yml ├── .gitignore ├── CODEOWNERS ├── LICENSE ├── Makefile ├── README.md ├── dev-resources └── Makefile.i18n ├── dev ├── java-env └── run-test ├── ext └── test │ └── manage-tinyfs ├── locales └── messages.pot ├── project.clj ├── src └── puppetlabs │ └── stockpile │ └── queue.clj └── test └── puppetlabs └── stockpile └── queue_test.clj /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | 2 | name: main 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lein-test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | with: 11 | persist-credentials: false 12 | - uses: actions/setup-java@v3 13 | with: 14 | distribution: 'temurin' 15 | java-version: | 16 | 8 17 | 11 18 | 17 19 | - uses: actions/cache@v3 20 | with: 21 | key: apt-${{ matrix.jdk }} 22 | path: | 23 | ~/.m2/repository 24 | /var/cache/apt/archives/*.deb 25 | - run: sudo -i apt-get update 26 | - run: sudo -i apt-get -y install leiningen 27 | - name: lein test (jdk 8) 28 | run: dev/java-env --expect-major 8 "${JAVA_HOME_8_X64}" dev/run-test --use-sudo 29 | - name: lein test (jdk 11) 30 | run: dev/java-env --expect-major 11 "${JAVA_HOME_11_X64}" dev/run-test --use-sudo 31 | - name: lein test (jdk 17) 32 | run: dev/java-env --expect-major 17 "${JAVA_HOME_17_X64}" dev/run-test --use-sudo 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /.lein-* 3 | /.nrepl-port 4 | /checkouts/ 5 | /target/ 6 | 7 | # clj-i18n 8 | /mp-* 9 | /resources/locales.clj 10 | /resources/puppetlabs/stockpile/*.class 11 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @puppetlabs/dumpling 2 | * @puppetlabs/skeletor 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include dev-resources/Makefile.i18n 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # stockpile [![Clojars Project](https://img.shields.io/clojars/v/puppetlabs/stockpile.svg)](https://clojars.org/puppetlabs/stockpile) ![main](https://github.com/puppetlabs/stockpile/workflows/main/badge.svg) 2 | 3 | A simple, durable Clojure queueing library. While this is believed to 4 | be reasonably solid, it is still relatively new, and the API or 5 | behavior may change without warning. 6 | 7 | Stockpile supports the durable storage and retrieval of data. After 8 | storage, stockpile returns an `entry` that can be used to access the 9 | data later, and when no longer needed, the data can be atomically 10 | `discard`ed. 11 | 12 | Stockpile is explicitly designed to keep minimal state outside the 13 | filesystem. After opening a queue, you can call `reduce` to traverse 14 | the existing entries, but stockpile itself does not retain information 15 | about the entries. You must preserve any that you might want to 16 | access later. 17 | 18 | The ordering of any two entries can be compared (by `id`), but that 19 | ordering is not guaranteed to be exact, only some approximation of 20 | their relative insertion order. 21 | 22 | A metadata string can be provided for each item stored, but the length 23 | of that string may be limited by the underlying filesystem. (The 24 | string is currently encoded into the queue item's pathname so that it 25 | can be retrieved without having to open or read the file itself). The 26 | filesystem might also alter the metadata in other ways, for example if 27 | it does not preserve case. The path that's ultimately specified to 28 | the filesystem by the JVM may be affected by the locale, and on Linux 29 | with common filesystems, for example, often produces UTF-8 paths. See 30 | the queue `store` docstring for further information. 31 | 32 | Stockpile is intended to work correctly on any filesystem where rename 33 | (ATOMIC\_MOVE) works correctly, and where calling fsync/fdatasync on a 34 | file and its parent directory makes the file durable. In the past 35 | (before JDK 9), [that did not include OS X](https://bugs.openjdk.java.net/browse/JDK-8080589). 36 | 37 | And there's apparently a [controversy](http://mail.openjdk.java.net/pipermail/nio-dev/2015-May/003140.html) 38 | about whether to continue to support the undocumented method stockpile 39 | uses to sync the parent directory, or to 40 | [provide some other documented mechanism](https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8080235). 41 | 42 | Unless the items being inserted into the queue are large enough for 43 | the sequential transfer rate to dominate, the insertion rate is likely 44 | to be limited by the maximum "fsync/fdatasync rate" of the underlying 45 | filesystem. 46 | 47 | The current implementation tracks the queue ids using an AtomicLong, 48 | and while that counter could overflow, even at 100,000 stores per 49 | second, it should require about 290 million years. 50 | 51 | Stockpile's behavior given unexpected files inside its directory is 52 | undefined. 53 | 54 | ## Usage 55 | 56 | See queue.clj and queue_test.clj for the API documentation and sample 57 | usage. 58 | 59 | ## Testing 60 | 61 | As expected, "lein test" will run the test suite, but there are some 62 | additional tests can only be run if `STOCKPILE_TINY_TEST_FS` is set 63 | to a directory that resides on an otherwise quiet filesystem with less 64 | than 10MB of free space, that is not the current filesystem: 65 | 66 | STOCKPILE_TINY_TEST_FS=~/tmp/tiny lein test 67 | 68 | During the tests stockpile may repeatedly fill that filesystem. 69 | 70 | You can also run the tests via `dev/run-test`, and if you're on 71 | Linux and root, it will automatically set up, use, and tear down a 72 | suitable tiny loopback filesystem. Alternately, if you have sudo 73 | access to root, you can invoke `dev/run-test --use-sudo` to do the 74 | same. Though invoking `dev/run-test` outside of a "throwaway" 75 | virtual machine is not recommended right now. 76 | 77 | ## Implementation notes 78 | 79 | The current implementation follows the the traditional POSIX-oriented 80 | approach of storing each entry in its own file, with a name that's 81 | guaranteed to be unique, and where durability is provided by this 82 | sequence of operations: 83 | 84 | - Write the entry data to a temp file, 85 | - fdatasync() the temp file, 86 | - rename() the temp file to the final name, 87 | - and fsync() the parent directory. 88 | 89 | Or rather, stockpile calls JVM functions that are believed to provide 90 | that behavior or an equivalent result. The parent directory fsync() 91 | is required in order to ensure that the destination file name is also 92 | durable. 93 | 94 | Given this approach, the "fsync rate" of the underlying filesystem 95 | will constrain the entry storage rate for a given queue, and so 96 | batching multiple items into a single entry (when feasible) may be an 97 | effective way to increase performance. 98 | 99 | ### An overview of relevant concepts: 100 | 101 | - http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/ 102 | 103 | ### Related JVM methods and documentation: 104 | 105 | - https://docs.oracle.com/javase/7/docs/api/java/nio/channels/FileChannel.html#force(boolean) 106 | - https://docs.oracle.com/javase/7/docs/api/java/nio/file/Files.html#move(java.nio.file.Path,%20java.nio.file.Path,%20java.nio.file.CopyOption...) 107 | 108 | ### Supporting POSIX functions: 109 | 110 | - http://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html 111 | - http://pubs.opengroup.org/onlinepubs/9699919799/functions/fdatasync.html 112 | - http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html 113 | 114 | ## License 115 | 116 | Copyright © 2016 Puppet Labs Inc 117 | 118 | Distributed under the Apache License Version 2.0. See ./LICENSE for 119 | details. 120 | -------------------------------------------------------------------------------- /dev-resources/Makefile.i18n: -------------------------------------------------------------------------------- 1 | # -*- Makefile -*- 2 | # This file was generated by the i18n leiningen plugin 3 | # Do not edit this file; it will be overwritten the next time you run 4 | # lein i18n init 5 | # 6 | 7 | # The locale in which our messages are written, and for which we therefore 8 | # have messages without any further effort 9 | MESSAGE_LOCALE=en 10 | 11 | # The name of the package into which the translations bundle will be placed 12 | BUNDLE=puppetlabs.stockpile 13 | # The list of names of packages covered by the translation bundle; 14 | # by default it contains a single package - the same where the translations 15 | # bundle itself is placed - but this can be overridden - preferably in 16 | # the top level Makefile 17 | PACKAGES?=$(BUNDLE) 18 | LOCALES=$(basename $(notdir $(wildcard locales/*.po))) 19 | BUNDLE_DIR=$(subst .,/,$(BUNDLE)) 20 | BUNDLE_FILES=$(patsubst %,resources/$(BUNDLE_DIR)/Messages_%.class,$(MESSAGE_LOCALE) $(LOCALES)) 21 | FIND_SOURCES=find src -name \*.clj 22 | # xgettext before 0.19 does not understand --add-location=file. Even CentOS 23 | # 7 ships with an older gettext. We will therefore generate full location 24 | # info on those systems, and only file names where xgettext supports it 25 | LOC_OPT=$(shell xgettext --add-location=file -f - /dev/null 2>&1 && echo --add-location=file || echo --add-location) 26 | 27 | LOCALES_CLJ=resources/locales.clj 28 | define LOCALES_CLJ_CONTENTS 29 | { 30 | :locales #{$(patsubst %,"%",$(MESSAGE_LOCALE) $(LOCALES))} 31 | :packages [$(patsubst %,"%",$(PACKAGES))] 32 | :bundle $(patsubst %,"%",$(BUNDLE).Messages) 33 | } 34 | endef 35 | export LOCALES_CLJ_CONTENTS 36 | 37 | 38 | i18n: update-pot msgfmt 39 | 40 | # Update locales/messages.pot 41 | update-pot: locales/messages.pot 42 | 43 | locales/messages.pot: $(shell $(FIND_SOURCES)) | locales 44 | @tmp=$$(mktemp $@.tmp.XXXX); \ 45 | $(FIND_SOURCES) \ 46 | | xgettext --from-code=UTF-8 --language=lisp \ 47 | --copyright-holder='Puppet ' \ 48 | --package-name="$(BUNDLE)" \ 49 | --package-version="$(BUNDLE_VERSION)" \ 50 | --msgid-bugs-address="docs@puppet.com" \ 51 | -k \ 52 | -kmark:1 -ki18n/mark:1 \ 53 | -ktrs:1 -ki18n/trs:1 \ 54 | -ktru:1 -ki18n/tru:1 \ 55 | -ktrun:1,2 -ki18n/trun:1,2 \ 56 | -ktrsn:1,2 -ki18n/trsn:1,2 \ 57 | $(LOC_OPT) \ 58 | --add-comments --sort-by-file \ 59 | -o $$tmp -f -; \ 60 | sed -i.bak -e 's/charset=CHARSET/charset=UTF-8/' $$tmp; \ 61 | sed -i.bak -e 's/POT-Creation-Date: [^\\]*/POT-Creation-Date: /' $$tmp; \ 62 | rm -f $$tmp.bak; \ 63 | if ! diff -q -I POT-Creation-Date $$tmp $@ >/dev/null 2>&1; then \ 64 | mv $$tmp $@; \ 65 | else \ 66 | rm $$tmp; touch $@; \ 67 | fi 68 | 69 | # Run msgfmt over all .po files to generate Java resource bundles 70 | # and create the locales.clj file 71 | msgfmt: $(BUNDLE_FILES) $(LOCALES_CLJ) 72 | 73 | # force rebuild of locales.clj if its contents is not the 74 | # the desired one 75 | ifneq ($(shell cat $(LOCALES_CLJ) 2> /dev/null),$(shell echo '$(subst ','\'',$(LOCALES_CLJ_CONTENTS))')) 76 | .PHONY: $(LOCALES_CLJ) 77 | endif 78 | $(LOCALES_CLJ): | resources 79 | @echo "Writing $@" 80 | @echo "$$LOCALES_CLJ_CONTENTS" > $@ 81 | 82 | resources/$(BUNDLE_DIR)/Messages_%.class: locales/%.po | resources 83 | msgfmt --java2 -d resources -r $(BUNDLE).Messages -l $(*F) $< 84 | 85 | resources/$(BUNDLE_DIR)/Messages_$(MESSAGE_LOCALE).class: locales/messages.pot | resources 86 | msgfmt --java2 -d resources -r $(BUNDLE).Messages -l $(MESSAGE_LOCALE) $< 87 | 88 | # Translators use this when they update translations; this copies any 89 | # changes in the pot file into their language-specific po file 90 | locales/%.po: locales/messages.pot 91 | @if [ -f $@ ]; then \ 92 | msgmerge -U $@ $< && touch $@; \ 93 | else \ 94 | touch $@ && msginit --no-translator -l $(*F) -o $@ -i $<; \ 95 | fi 96 | 97 | resources locales: 98 | @mkdir $@ 99 | 100 | help: 101 | $(info $(HELP)) 102 | @echo 103 | 104 | .PHONY: help 105 | 106 | define HELP 107 | This Makefile assists in handling i18n related tasks during development. Files 108 | that need to be checked into source control are put into the locales/ directory. 109 | They are 110 | 111 | locales/messages.pot - the POT file generated by 'make update-pot' 112 | locales/$$LANG.po - the translations for $$LANG 113 | 114 | Only the $$LANG.po files should be edited manually; this is usually done by 115 | translators. 116 | 117 | You can use the following targets: 118 | 119 | i18n: refresh all the files in locales/ and recompile resources 120 | update-pot: extract strings and update locales/messages.pot 121 | locales/LANG.po: refresh or create translations for LANG 122 | msgfmt: compile the translations into Java classes; this step is 123 | needed to make translations available to the Clojure code 124 | and produces Java class files in resources/ 125 | endef 126 | # @todo lutter 2015-04-20: for projects that use libraries with their own 127 | # translation, we need to combine all their translations into one big po 128 | # file and then run msgfmt over that so that we only have to deal with one 129 | # resource bundle 130 | -------------------------------------------------------------------------------- /dev/java-env: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ueo pipefail 4 | 5 | usage() 6 | { 7 | echo "Usage: java-env [--expect-major N] JAVA_HOME [--] CMD [ARG ...]"; 8 | } 9 | 10 | misuse() { usage 1>&2; exit 2; } 11 | 12 | expect_maj='' 13 | 14 | while test $# -gt 0; do 15 | case "$1" in 16 | --expect-major) 17 | shift 18 | test $# -gt 1 || misuse 19 | expect_maj="$1" 20 | shift 21 | ;; 22 | -*|--*) misuse; break ;; 23 | --|*) break ;; 24 | esac 25 | done 26 | 27 | test $# -gt 1 || misuse 28 | java_home="$1" 29 | shift 30 | 31 | export JAVA_HOME="$java_home" 32 | PATH="$JAVA_HOME/bin:$PATH" 33 | 34 | if test "$expect_maj"; then 35 | if test "$expect_maj" -lt 10; then 36 | ver="$(java -version 2>&1 | head -1)" 37 | if ! [[ "$ver" =~ ^openjdk\ version\ \"1\.$expect_maj(\.|$) ]] ; then 38 | echo "error: expected jdk $expect_maj, not $ver" 1>&2 39 | exit 2 40 | fi 41 | else 42 | ver="$(java --version 2>&1 | head -1)" 43 | if ! [[ "$ver" =~ ^openjdk\ $expect_maj(\.|$) ]] ; then 44 | echo "error: expected jdk $expect_maj, not $ver" 1>&2 45 | exit 2 46 | fi 47 | fi 48 | fi 49 | 50 | exec "$@" 51 | -------------------------------------------------------------------------------- /dev/run-test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | # Make sure we're in the right place 6 | test -x ext/test/manage-tinyfs 7 | test -f src/puppetlabs/stockpile/queue.clj 8 | 9 | manage_tinyfs="$(pwd)/ext/test/manage-tinyfs" 10 | 11 | usage() 12 | { 13 | echo "Usage: $0 [--use-sudo|--no-sudo] [-- LEIN_TEST_ARGS]" 14 | } 15 | 16 | tmpdir='' 17 | destroy_tinyfs='' 18 | 19 | clean-up() 20 | { 21 | if test "$destroy_tinyfs"; then 22 | as-root "$manage_tinyfs" destroy "$tmpdir" 23 | fi 24 | if test "$tmpdir"; then 25 | rm -rf "$tmpdir" 26 | fi 27 | } 28 | 29 | use_sudo='' 30 | 31 | while test "$#" -ne 0; do 32 | case "$1" in 33 | --use-sudo) 34 | shift 35 | use_sudo=true 36 | ;; 37 | --no-sudo) 38 | shift 39 | use_sudo='' 40 | ;; 41 | --) 42 | shift 43 | break 44 | ;; 45 | *) 46 | usage 1>&2 47 | exit 1 48 | ;; 49 | esac 50 | done 51 | 52 | test_tinyfs='' 53 | 54 | if test "$(uname -s)" != Linux; then 55 | echo "Not on Linux; skipping root tests" 1>&2 56 | else 57 | if test "$use_sudo"; then 58 | as-root() 59 | { 60 | sudo -i "$@" 61 | } 62 | test_tinyfs=true 63 | elif test "$(id -u)" = 0; then 64 | as-root() 65 | { 66 | "$@" 67 | } 68 | test_tinyfs=true 69 | else 70 | echo "Not root, and --use-sudo not specified; skipping root tests" 1>&2 71 | fi 72 | fi 73 | 74 | trap clean-up EXIT 75 | 76 | if [ "$test_tinyfs" ]; then 77 | mkdir -p target 78 | tmpdir="$(mktemp -d "$(pwd)/target/run-test-XXXXXXX")" 79 | destroy_tinyfs=true 80 | as-root "$manage_tinyfs" create "$tmpdir" "$(id -u)" "$(id -g)" 81 | export STOCKPILE_TINY_TEST_FS="$tmpdir/tinyfs" 82 | fi 83 | 84 | java -version 85 | # Don't exec or the trap won't fire 86 | lein test "$@" 87 | -------------------------------------------------------------------------------- /ext/test/manage-tinyfs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | usage() 6 | { 7 | echo "Usage: $0 create PARENT_TMPDIR UID GID" 8 | echo " $0 destroy PARENT_TMPDIR" 9 | } 10 | 11 | create-tinyfs() 12 | { 13 | local tmpdir="$1" uid="$2" gid="$3" origdir 14 | origdir="$(pwd)" 15 | cd -P "$tmpdir" 16 | dd if=/dev/zero of=tinyfs.img bs=10MB count=1 17 | chmod 600 tinyfs.img 18 | mke2fs -F tinyfs.img 19 | mkdir tinyfs 20 | mount -oloop tinyfs.img tinyfs 21 | chown "$uid:$gid" tinyfs 22 | cd "$origdir" 23 | } 24 | 25 | destroy-tinyfs() 26 | { 27 | local tmpdir="$1" 28 | umount -l -d "$tmpdir"/tinyfs 29 | rm -f "$tmpdir"/tinyfs.img 30 | } 31 | 32 | set -x 33 | 34 | case "$1" in 35 | create) 36 | shift 37 | if [ "$#" -ne 3 ]; then 38 | usage 1>&2 39 | exit 1 40 | fi 41 | create-tinyfs "$@" 42 | ;; 43 | destroy) 44 | shift 45 | if [ "$#" -ne 1 ]; then 46 | usage 1>&2 47 | exit 1 48 | fi 49 | destroy-tinyfs "$1" 50 | ;; 51 | *) 52 | usage 1>&2 53 | exit 1 54 | esac 55 | -------------------------------------------------------------------------------- /locales/messages.pot: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) YEAR Puppet 3 | # This file is distributed under the same license as the puppetlabs.stockpile package. 4 | # FIRST AUTHOR , YEAR. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: puppetlabs.stockpile \n" 10 | "Report-Msgid-Bugs-To: docs@puppet.com\n" 11 | "POT-Creation-Date: \n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "Language: \n" 16 | "MIME-Version: 1.0\n" 17 | "Content-Type: text/plain; charset=UTF-8\n" 18 | "Content-Transfer-Encoding: 8bit\n" 19 | 20 | #: src/puppetlabs/stockpile/queue.clj 21 | msgid "id is not an integer: {0}" 22 | msgstr "" 23 | 24 | #: src/puppetlabs/stockpile/queue.clj 25 | msgid "metadata is not a string: {0}" 26 | msgstr "" 27 | 28 | #: src/puppetlabs/stockpile/queue.clj 29 | msgid "Invalid queue token {0} found in {1}" 30 | msgstr "" 31 | 32 | #: src/puppetlabs/stockpile/queue.clj 33 | msgid "unable to delete temp file {0} after error" 34 | msgstr "" 35 | 36 | #: src/puppetlabs/stockpile/queue.clj 37 | msgid "unable to commit; leaving stream data in {0}" 38 | msgstr "" 39 | 40 | #: src/puppetlabs/stockpile/queue.clj 41 | msgid "No file found for entry {0} at {1}" 42 | msgstr "" 43 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject puppetlabs/stockpile "0.1.0-SNAPSHOT" 2 | :description "Simple, durable Clojure queuing library" 3 | :url "https://github.com/puppetlabs/stockpile" 4 | :license {:name "Apache License Version 2.0" 5 | :url "http://www.apache.org/licenses/LICENSE-2.0"} 6 | :dependencies [[puppetlabs/i18n "0.4.3"] 7 | [org.clojure/clojure "1.8.0"]] 8 | :plugins [[puppetlabs/i18n "0.4.3"]] 9 | :profiles {:dev {:dependencies [[org.apache.commons/commons-lang3 "3.4"]]}} 10 | 11 | :deploy-repositories [["releases" {:url "https://clojars.org/repo" 12 | :username :env/clojars_jenkins_username 13 | :password :env/clojars_jenkins_password 14 | :sign-releases false}]]) 15 | -------------------------------------------------------------------------------- /src/puppetlabs/stockpile/queue.clj: -------------------------------------------------------------------------------- 1 | (ns puppetlabs.stockpile.queue 2 | (:refer-clojure :exclude [reduce]) 3 | (:require 4 | [puppetlabs.i18n.core :refer [trs]]) 5 | (:import 6 | [clojure.lang BigInt] 7 | [java.io ByteArrayInputStream File FileOutputStream InputStream] 8 | [java.nio.file AtomicMoveNotSupportedException DirectoryStream 9 | FileSystemException NoSuchFileException Path Paths] 10 | [java.nio.channels FileChannel] 11 | [java.nio.file FileAlreadyExistsException Files OpenOption StandardCopyOption] 12 | [java.nio.file.attribute FileAttribute] 13 | [java.util.concurrent.atomic AtomicLong])) 14 | 15 | ;; Queue structure: 16 | ;; - qdir/stockpile 17 | ;; - qdir/q/INTEGER # message 18 | ;; - qdir/q/INTEGER-ENCODED_METADATA # message 19 | ;; - qdir/q/tmp-BLARG # pending message 20 | 21 | (defn- basename [^Path path] 22 | (.getName path (dec (.getNameCount path)))) 23 | 24 | (defn ^Path path-get [^String s & more-strings] 25 | (Paths/get s (into-array String more-strings))) 26 | 27 | (defn- parse-integer [x] 28 | (try 29 | (Long/parseLong x) 30 | (catch NumberFormatException ex 31 | nil))) 32 | 33 | (defprotocol AsPath 34 | (as-path ^Path [x])) 35 | 36 | (extend-protocol AsPath 37 | Path 38 | (as-path [x] x) 39 | String 40 | (as-path [x] (path-get x)) 41 | File 42 | (as-path [x] (.toPath x))) 43 | 44 | (defprotocol Entry 45 | (entry-id [entry]) 46 | (entry-meta [entry])) 47 | 48 | (defrecord MetaEntry [id metadata] 49 | Entry 50 | (entry-id [this] id) 51 | (entry-meta [this] metadata)) 52 | 53 | (extend-protocol Entry 54 | Long 55 | (entry-id [this] this) 56 | (entry-meta [this] nil)) 57 | 58 | (defn- create-tmp-file [parent] 59 | ;; Don't change the prefix/suffix here casually. Other 60 | ;; code below assumes, for example, that a temporary file will never 61 | ;; be named "stockpile". 62 | (Files/createTempFile (as-path parent) "tmp-" "" 63 | (into-array FileAttribute []))) 64 | 65 | (defn fsync [x metadata?] 66 | (with-open [fc (FileChannel/open (as-path x) 67 | (into-array OpenOption []))] 68 | (.force fc metadata?))) 69 | 70 | (def ^:private copt-atomic StandardCopyOption/ATOMIC_MOVE) 71 | (def ^:private copt-replace StandardCopyOption/REPLACE_EXISTING) 72 | (def ^:private copts-type (class (into-array StandardCopyOption []))) 73 | 74 | (defn ^copts-type copts [opts] 75 | (into-array StandardCopyOption opts)) 76 | 77 | (defn- atomic-move [src dest] 78 | (Files/move (as-path src) (as-path dest) 79 | (copts [copt-atomic]))) 80 | 81 | (defn- rename-durably 82 | "If possible, atomically renames src to dest (each of which may be a 83 | File, Path, or String). If dest already exists, on some platforms 84 | the replacement will succeed, and on others it will throw an 85 | IOException. The rename may also fail with 86 | AtomicMoveNotSupportedException (perhaps if src and dest are on 87 | different filesystems). See java.nio.file.Files/move for additional 88 | information. fsyncs the dest parent directory to make the final 89 | rename durable unless sync-parent? is false (presumably the caller 90 | will ensure the sync)." 91 | [src dest sync-parent?] 92 | (atomic-move src dest) 93 | (when sync-parent? 94 | (fsync (.getParent (as-path dest)) true))) 95 | 96 | (defn- delete-if-exists [path] 97 | ;; Solely exists for error handling tests 98 | (Files/deleteIfExists path)) 99 | 100 | (defn- write-stream [^InputStream stream ^Path dest] 101 | ;; Solely exists for error handling tests 102 | (Files/copy stream dest (copts [copt-replace]))) 103 | 104 | (defn- qpath ^Path [{:keys [^Path directory] :as q}] 105 | (.resolve directory "q")) 106 | 107 | (defn- queue-entry-path 108 | [q id metadata] 109 | (let [^Path parent (qpath q) 110 | ^String entry-name (apply str id (when metadata ["-" metadata]))] 111 | (.resolve parent entry-name))) 112 | 113 | (defn- entry-path 114 | [q entry] 115 | (queue-entry-path q (entry-id entry) (entry-meta entry))) 116 | 117 | (defn- filename->entry 118 | "Returns an entry if name can be parsed as such, i.e. either as 119 | an integer or integer-metadata, nil otherwise." 120 | [^String name] 121 | (let [dash (.indexOf name (int \-))] 122 | (if (= -1 dash) 123 | (parse-integer name) 124 | ;; Perhaps it has metadata 125 | (when-let [id (parse-integer (subs name 0 dash))] 126 | (->MetaEntry id (subs name (inc dash))))))) 127 | 128 | (defrecord Stockpile [directory next-likely-id]) 129 | 130 | (defn- reduce-paths 131 | [f val ^DirectoryStream dirstream] 132 | (with-open [_ dirstream] 133 | (clojure.core/reduce f val (-> dirstream .iterator iterator-seq)))) 134 | 135 | (defn- plausible-prefix? 136 | [s] 137 | (-> #"^[0-9](?:-.)?+" (.matcher s) .find)) 138 | 139 | 140 | ;;; Stable, public interface 141 | 142 | (defn entry [id metadata] 143 | (let [id (if (integer? id) 144 | (long id) 145 | (throw 146 | (IllegalArgumentException. 147 | (trs "id is not an integer: {0}" id))))] 148 | (cond 149 | (nil? metadata) id 150 | 151 | (not (string? metadata)) 152 | (throw 153 | (IllegalArgumentException. 154 | (trs "metadata is not a string: {0}" (pr-str metadata)))) 155 | 156 | :else (->MetaEntry id metadata)))) 157 | 158 | (defn next-likely-id 159 | "Returns a likely id for the next message stored in the q. No 160 | subsequent entry ids will be less than this value." 161 | [{^AtomicLong next :next-likely-id :as q}] 162 | (.get next)) 163 | 164 | (defn create 165 | "Creates a new queue in directory, which must not exist, and returns 166 | the queue. If an exception is thrown, the directory named may or 167 | may not exist and may or may not be empty." 168 | [directory] 169 | (let [top (as-path directory) 170 | q (.resolve top "q")] 171 | (Files/createDirectory top (into-array FileAttribute [])) 172 | (Files/createDirectory q (into-array FileAttribute [])) 173 | ;; This sentinel is last - indicates the queue is *ready* 174 | (let [tmp (create-tmp-file top)] 175 | (with-open [out (FileOutputStream. (.toFile tmp))] 176 | (.write out (.getBytes "0 stockpile" "UTF-8"))) 177 | (fsync tmp false) 178 | (rename-durably tmp (.resolve top "stockpile") false)) 179 | (fsync top true) 180 | (->Stockpile top (AtomicLong. 0)))) 181 | 182 | (defn open 183 | "Opens the queue in directory, and returns it. Expects only 184 | stockpile created files in the directory, and currently deletes any 185 | existing file in the queue whose name starts with \"tmp-\"." 186 | [directory] 187 | (let [top (as-path directory) 188 | q (.resolve top "q")] 189 | (let [info-file (.resolve top "stockpile") 190 | info (String. (Files/readAllBytes info-file) "UTF-8")] 191 | (when-not (= "0 stockpile" info) 192 | (throw (IllegalStateException. 193 | (trs "Invalid queue token {0} found in {1}" 194 | (pr-str info) 195 | (pr-str (str info-file))))))) 196 | (let [max-id (reduce-paths (fn [result ^Path p] 197 | (let [name (str (basename p))] 198 | (cond 199 | (.startsWith name "tmp-") 200 | (do (Files/deleteIfExists p) result) 201 | 202 | (plausible-prefix? name) 203 | (max result (-> name 204 | filename->entry 205 | entry-id)) 206 | 207 | :else 208 | result))) 209 | 0 210 | (Files/newDirectoryStream q))] 211 | (->Stockpile top (AtomicLong. (inc max-id)))))) 212 | 213 | (defn reduce 214 | "Calls (f reduction entry) for each existing entry as-per reduce, 215 | with val as the initial reduction, and returns the result. The 216 | ordering of the calls is unspecified, as is the effect of concurrent 217 | discards. The reduction may be escaped by throwing a unique 218 | exception (cf. slingshot). For example: (reduce \"foo\" conj [])." 219 | [q f val] 220 | (reduce-paths (fn [result ^Path p] 221 | (let [name (-> p basename str)] 222 | (if-not (plausible-prefix? name) 223 | result 224 | (f result (filename->entry name))))) 225 | val 226 | (Files/newDirectoryStream (qpath q)))) 227 | 228 | (defn store 229 | "Atomically and durably enqueues the content of stream, and returns 230 | an entry that can be used to refer to the content later. An ex-info 231 | exception of {:kind ::unable-to-commit :stream-data path} may be 232 | thrown if store was able to read the data from the stream, but 233 | unable to make it durable. If any other exception is thrown, the 234 | state of the stream is unknown. The :stream-data value will be a 235 | path to a file containing all of the data that was in the stream. 236 | Among other things, it's possible that ::unable-to-commit indicates 237 | the metadata was incompatible with the underlying filesystem (it was 238 | too long, couldn't be encoded, etc.). That's because the current 239 | implementation records the metadata in a file name corresponding to 240 | the entry, and may use up to 20 (Unicode Basic Latin block) 241 | characters of that file name for internal purposes. The remainder 242 | of the filename is available for the metadata, but the maximum 243 | length of that remainder depends on the platform and target 244 | filesystem. Many common filesystems now allow a file name to be up 245 | to 255 characters or bytes, and at least on Linux, the JVM converts 246 | the Unicode string path to a filesystem path using an encoding that 247 | depends on the locale, often choosing UTF-8. So assuming a UTF-8 248 | encoding and a 255 byte maximum path length (e.g. ext4), after 249 | subtracting the 20 (UTF-8 encoded Basic Latin block) bytes reserved 250 | for internal use, there may be up to 235 bytes available for the 251 | metadata. Of course how many Unicode characters that will allow 252 | depends on their size when converted to UTF-8. Whenever there's an 253 | error, it's possible that the attempt to clean up may fail and leave 254 | behind a temporary file. In that case create throws an ex-info 255 | exception of {:kind ::path-cleanup-failure-after-error :path 256 | p :exception ex} with the exception produced by the original failure 257 | as the cause. To handle that possibility, callers may want to 258 | structure invocations with a nested try like this: 259 | (try 260 | (try+ 261 | (stock/create ...) 262 | (catch [:kind ::path-cleanup-failure-after-error] 263 | {:keys [path exception]} 264 | ;; Perhaps log or try to clean up the path more aggressively 265 | (throw (:cause &throw-context))) 266 | (catch SomeExceptionThatCausedCreateToFail 267 | ;; Reached for this exception whether or not there was a 268 | ;; cleanup failure 269 | ...) 270 | ...) 271 | Additionally, among other exceptions, the current implementation may 272 | throw any documented by java.nio.file.Files/move for an ATOMIC_MOVE 273 | within the same directory." 274 | ([q stream] (store q stream nil)) 275 | ([q ^ByteArrayInputStream stream metadata] 276 | (let [^AtomicLong next (:next-likely-id q) 277 | qd (qpath q)] 278 | (let [^Path tmp-dest (create-tmp-file qd)] 279 | ;; It might be possible to optimize some cases with 280 | ;; transferFrom/transferTo eventually. 281 | (try 282 | (write-stream stream tmp-dest) 283 | (catch Exception ex 284 | (try 285 | (delete-if-exists tmp-dest) 286 | (catch Exception del-ex 287 | (throw 288 | (ex-info (trs "unable to delete temp file {0} after error" 289 | (pr-str (str tmp-dest))) 290 | {:kind ::path-cleanup-failure-after-error 291 | :path tmp-dest 292 | :exception del-ex} 293 | ex)))) 294 | (throw ex))) 295 | (try 296 | (fsync tmp-dest false) 297 | (loop [] 298 | (let [id (.getAndIncrement next) 299 | target (queue-entry-path q id metadata) 300 | ;; Can't recur from catch 301 | moved? (try 302 | (rename-durably tmp-dest target true) 303 | true 304 | (catch FileAlreadyExistsException ex 305 | false))] 306 | (if moved? 307 | (entry id metadata) 308 | (recur)))) 309 | (catch Exception ex 310 | (throw (ex-info (trs "unable to commit; leaving stream data in {0}" 311 | (pr-str (str tmp-dest))) 312 | {:kind ::unable-to-commit 313 | :stream-data tmp-dest} 314 | ex)))))))) 315 | 316 | (defn stream 317 | "Returns an unbuffered stream of the entry's data. Throws an 318 | ex-info exception of {:kind ::no-such-entry :entry e :source s} if 319 | the requested entry does not exist. Currently the :source will 320 | always be a Path." 321 | [q entry] 322 | (let [path (entry-path q entry)] 323 | (try 324 | (Files/newInputStream path (make-array OpenOption 0)) 325 | (catch NoSuchFileException ex 326 | (let [m (entry-meta entry) 327 | id (entry-id entry)] 328 | (throw (ex-info (trs "No file found for entry {0} at {1}" 329 | (if-not m id (pr-str [id m])) 330 | (pr-str (str path))) 331 | {:kind ::no-such-entry :entry entry :source path} 332 | ex))))))) 333 | 334 | (defn discard 335 | "Atomically and durably discards the entry (returned by store) from 336 | the queue. The discarded data will be placed at the destination 337 | path (durably if possible), when one is provided. This should be 338 | much more efficient, and likely safer if the destination is at least 339 | on the same filesystem as the queue. The results of calling this 340 | more than once for a given entry are undefined." 341 | ;; Not entirely certain the queue parent dir syncs are necessary *if* 342 | ;; everyone guarantees that you either see the file or not, and if 343 | ;; we're OK with the possibility of spurious redelivery. 344 | ([q entry] 345 | (Files/deleteIfExists (entry-path q entry)) 346 | (fsync (qpath q) true)) 347 | ([q entry destination] 348 | (let [^Path src (entry-path q entry) 349 | ^Path destination (as-path destination) 350 | moved? (try 351 | (Files/move src destination (copts [copt-atomic])) 352 | true 353 | (catch UnsupportedOperationException ex 354 | false) 355 | (catch AtomicMoveNotSupportedException ex 356 | false))] 357 | (when-not moved? 358 | (Files/copy src destination (copts [copt-replace])) 359 | (Files/delete src)) 360 | (fsync (.getParent destination) true) 361 | (fsync (qpath q) true)))) 362 | -------------------------------------------------------------------------------- /test/puppetlabs/stockpile/queue_test.clj: -------------------------------------------------------------------------------- 1 | (ns puppetlabs.stockpile.queue-test 2 | (:require [puppetlabs.stockpile.queue :as stock] 3 | [clojure.java.io :as io] 4 | [clojure.java.shell :as shell] 5 | [clojure.test :refer :all]) 6 | (:import 7 | [org.apache.commons.lang3 RandomStringUtils] 8 | [java.io ByteArrayInputStream File IOException] 9 | [java.nio.file Files NoSuchFileException OpenOption Path StandardOpenOption] 10 | [java.nio.file.attribute FileAttribute] 11 | [puppetlabs.stockpile.queue MetaEntry])) 12 | 13 | (defn relativize-file [wrt-path f] 14 | (.relativize wrt-path (.toPath f))) 15 | 16 | (defn relative-pathstr-seq [parent] 17 | (map #(str (relativize-file parent %)) 18 | (file-seq (.toFile parent)))) 19 | 20 | (def small-test-fs 21 | (if-let [v (System/getenv "STOCKPILE_TINY_TEST_FS")] 22 | (stock/path-get v) 23 | (binding [*out* *err*] 24 | (println "STOCKPILE_TINY_TEST_FS not defined; skipping related tests") 25 | false))) 26 | 27 | (defn random-path-segment [n] 28 | (loop [s (RandomStringUtils/random n)] 29 | (if (and (= -1 (.indexOf s java.io.File/separator)) 30 | (= -1 (.indexOf s (int \u0000)))) 31 | s 32 | (recur (RandomStringUtils/random n))))) 33 | 34 | (defn rm-r [pathstr] 35 | ;; Life's too short... 36 | (let [rm (shell/sh "rm" "-r" pathstr)] 37 | (when-not (zero? (:exit rm)) 38 | (throw (-> "'rm -r %s' failed: %s" 39 | (format (pr-str pathstr) (pr-str rm)) 40 | Exception.))))) 41 | 42 | (defn call-with-temp-dir-path 43 | [f] 44 | (let [tempdir (Files/createTempDirectory (.toPath (File. "target")) 45 | "stockpile-test-" 46 | (into-array FileAttribute [])) 47 | tempdirstr (str (.toAbsolutePath tempdir)) 48 | result (try 49 | (f (.toAbsolutePath tempdir)) 50 | (catch Exception ex 51 | (binding [*out* *err*] 52 | (println "Error: leaving temp dir" tempdirstr)) 53 | (throw ex)))] 54 | (rm-r tempdirstr) 55 | result)) 56 | 57 | (defn entry-path [q entry] 58 | (#'stock/queue-entry-path q (stock/entry-id entry) (stock/entry-meta entry))) 59 | 60 | (defn slurp-entry [q entry] 61 | (slurp (stock/stream q entry))) 62 | 63 | (defn store-str 64 | ([q s] 65 | (let [ent (stock/store q (-> s (.getBytes "UTF-8") ByteArrayInputStream.)) 66 | id (stock/entry-id ent)] 67 | (is (integer? ent)) 68 | (is (integer? id)) 69 | (is (not (stock/entry-meta ent))) 70 | ent)) 71 | ([q s metadata] 72 | (let [ent (stock/store q 73 | (-> s (.getBytes "UTF-8") ByteArrayInputStream.) 74 | metadata) 75 | id (stock/entry-id ent) 76 | meta (stock/entry-meta ent)] 77 | (is (integer? id)) 78 | (if metadata 79 | (do 80 | (is (instance? MetaEntry ent)) 81 | (is (= metadata (stock/entry-meta ent)))) 82 | (is (not (stock/entry-meta ent)))) 83 | ent))) 84 | 85 | (defn purge-queue [qdir] 86 | (let [q (stock/open qdir) 87 | entries (stock/reduce q conj ())] 88 | (doseq [entry entries] 89 | (stock/discard entry)))) 90 | 91 | (deftest bad-entries 92 | (is (thrown? IllegalArgumentException (stock/entry "foo"))) 93 | (is (thrown? IllegalArgumentException (stock/entry "foo" 1))) 94 | (is (thrown? IllegalArgumentException (stock/entry 1 2)))) 95 | 96 | (deftest entry-ids 97 | (call-with-temp-dir-path 98 | (fn [tmpdir] 99 | ;; Expectations specific to the current implementation 100 | (let [q (stock/create (.toFile (.resolve tmpdir "queue")))] 101 | (is (zero? (stock/next-likely-id q))) 102 | (let [e (store-str q "first")] 103 | (is (zero? (stock/entry-id e))) 104 | (is (= 1 (stock/next-likely-id q))) 105 | (let [e (store-str q "second")] 106 | (is (= 1 (stock/entry-id e))) 107 | (is (= 2 (stock/next-likely-id q))))))))) 108 | 109 | (deftest basics 110 | (call-with-temp-dir-path 111 | (fn [tmpdir] 112 | (let [q (stock/create (.toFile (.resolve tmpdir "queue")))] 113 | (let [entry-1 (store-str q "foo") 114 | entry-2 (store-str q "bar" "*so* meta") 115 | id-1 (stock/entry-id entry-1) 116 | id-2 (stock/entry-id entry-2)] 117 | (is (< id-1 id-2)) 118 | (is (> id-2 id-1)) 119 | (is (= "foo" (slurp-entry q entry-1))) 120 | (is (= "bar" (slurp-entry q entry-2))) 121 | 122 | (stock/discard q entry-1) 123 | (is (= "bar" (slurp-entry q entry-2))) 124 | (try 125 | (slurp-entry q entry-1) 126 | (catch Exception ex 127 | (= {:entry entry-1 :source (entry-path q entry-1)} 128 | (ex-data ex)))))) 129 | (is (= #{"" "queue" "queue/q" "queue/q/1-*so* meta" "queue/stockpile"} 130 | (set (relative-pathstr-seq tmpdir))))))) 131 | 132 | (deftest basic-persistence 133 | ;; Some of the validation is handled implicitly by store-str 134 | (call-with-temp-dir-path 135 | (fn [tmpdir] 136 | (let [qdir (.toFile (.resolve tmpdir "queue")) 137 | reduction-0 (stock/reduce (stock/create qdir) 138 | #(throw (Exception. "unexpected")) :empty) 139 | ent-1 (-> (stock/open qdir) (store-str "foo")) 140 | ent-1-id (stock/entry-id ent-1) 141 | reduction-1 (stock/reduce (stock/open qdir) conj #{})] 142 | 143 | (is (= :empty reduction-0)) 144 | 145 | ;; Check first reduction (should be one element) 146 | (is (= #{ent-1} reduction-1)) 147 | (is (= #{(stock/entry ent-1-id nil)} reduction-1)) 148 | (let [ent (first reduction-1)] 149 | (is (= ent-1-id (stock/entry-id ent))) 150 | (is (not (stock/entry-meta ent)))) 151 | 152 | (is (= "foo" (slurp-entry (stock/open qdir) ent-1))) 153 | 154 | (let [ent-2 (-> (stock/open qdir) (store-str "bar" "meta bar")) 155 | ent-2-id (stock/entry-id ent-2) 156 | reduction-2 (stock/reduce (stock/open qdir) conj #{})] 157 | 158 | ;; Check second reduction (should be two elements) 159 | (is (= #{ent-1 ent-2} reduction-2)) 160 | (is (= #{(stock/entry ent-1-id nil) 161 | (stock/entry ent-2-id "meta bar")} 162 | reduction-2)) 163 | (let [ent (get reduction-2 ent-2)] 164 | (is (= ent-2-id (stock/entry-id ent))) 165 | (is (= "meta bar" (stock/entry-meta ent)))) 166 | 167 | (let [q (stock/open qdir)] 168 | (is (= "foo" (slurp-entry q ent-1))) 169 | (is (= "bar" (slurp-entry q ent-2)))))) 170 | 171 | (is (= #{"" "queue" "queue/q" 172 | "queue/q/1" "queue/q/2-meta bar" 173 | "queue/stockpile"} 174 | (set (relative-pathstr-seq tmpdir))))))) 175 | 176 | (deftest entry-manipulation 177 | (call-with-temp-dir-path 178 | (fn [tmpdir] 179 | (let [qdir (.toFile (.resolve tmpdir "queue")) 180 | q (stock/create qdir) 181 | inputs (for [i (range 10)] [(str i) (str "meta-" i)]) 182 | entries (for [[data metadata] inputs] 183 | (store-str q data metadata))] 184 | (doall 185 | (map (fn [input entry] 186 | (let [id (stock/entry-id entry) 187 | metadata (stock/entry-meta entry) 188 | reconstituted (stock/entry id metadata)] 189 | (is (= entry reconstituted)) 190 | (is (= (first input) 191 | (slurp-entry q reconstituted))))) 192 | inputs 193 | entries)))))) 194 | 195 | (deftest cleanup-failure-after-store-failure 196 | (call-with-temp-dir-path 197 | (fn [tmpdir] 198 | (let [qdir (.toFile (.resolve tmpdir "queue")) 199 | delete-failed (Exception. "delete") 200 | write-failed (Exception. "write") 201 | q (stock/create qdir) 202 | ex (try 203 | (with-redefs [stock/delete-if-exists (fn [& args] 204 | (throw delete-failed)) 205 | stock/write-stream (fn [& args] 206 | (throw write-failed))] 207 | (store-str q "first")) 208 | (catch Exception ex 209 | ex)) 210 | data (ex-data ex)] 211 | (is (= ::stock/path-cleanup-failure-after-error (:kind data))) 212 | (is (= delete-failed (:exception data))) 213 | (is (instance? Path (:path data))) 214 | (is (.exists (.toFile (:path data)))) 215 | (is (= write-failed (.getCause ex))))))) 216 | 217 | (deftest streaming-missing-entry 218 | (call-with-temp-dir-path 219 | (fn [tmpdir] 220 | (let [qdir (.toFile (.resolve tmpdir "queue")) 221 | delete-failed (Exception. "delete") 222 | write-failed (Exception. "write") 223 | q (stock/create qdir) 224 | entry (stock/entry 0 nil) 225 | ex (try 226 | (stock/stream q entry) 227 | (catch Exception ex 228 | ex)) 229 | data (ex-data ex)] 230 | (is (= ::stock/no-such-entry (:kind data))) 231 | (is (= entry (:entry data))) 232 | (is (= (entry-path q entry) (:source data))) 233 | (is (not (.exists (.toFile (:source data))))) 234 | (is (instance? NoSuchFileException (.getCause ex))))))) 235 | 236 | (deftest commit-failure-during-store 237 | (call-with-temp-dir-path 238 | (fn [tmpdir] 239 | (let [qdir (.toFile (.resolve tmpdir "queue")) 240 | rename-failed (Exception. "rename") 241 | q (stock/create qdir) 242 | ex (try 243 | (with-redefs [stock/rename-durably (fn [& args] 244 | (throw rename-failed))] 245 | (store-str q "first")) 246 | (catch Exception ex 247 | ex)) 248 | data (ex-data ex)] 249 | (is (= ::stock/unable-to-commit (:kind data))) 250 | (is (instance? Path (:stream-data data))) 251 | (is (= "first" (slurp (.toFile (:stream-data data))))) 252 | (is (= rename-failed (.getCause ex))))))) 253 | 254 | (deftest meta-encoding-round-trip 255 | (call-with-temp-dir-path 256 | (fn [tmpdir] 257 | (let [qdir (.toFile (.resolve tmpdir "queue")) 258 | q (stock/create qdir) 259 | batch-size 100] 260 | (dotimes [i batch-size] 261 | ;; We need to use a very short length here to avoid falling 262 | ;; afoul of path length limits since 8 random unicode 263 | ;; chars could expand to say 36 encoded bytes. 264 | (let [metadata (random-path-segment (rand-int 8))] 265 | (store-str q metadata metadata))))))) 266 | 267 | (deftest existing-tmp-removal 268 | (call-with-temp-dir-path 269 | (fn [tmpdir] 270 | (let [qdir (.toFile (.resolve tmpdir "queue")) 271 | garbage (File. qdir "q/tmp-garbage")] 272 | (stock/create qdir) 273 | (io/copy "foo" (File. qdir "q/tmp-garbage")) 274 | (let [q (stock/open qdir) 275 | entries (stock/reduce q conj ())] 276 | (is (= [] entries)) 277 | (is (not (.exists garbage)))))))) 278 | 279 | (defn test-discard-entry-to [destination tmpdir q-name] 280 | (let [qdir (.resolve tmpdir q-name) 281 | newq (stock/create qdir)] 282 | (let [entry (store-str newq "foo") 283 | q (stock/open qdir) 284 | read-entries (stock/reduce q conj ())] 285 | (is (= [entry] read-entries)) 286 | (stock/discard q entry destination) 287 | (is (= "foo" (String. (Files/readAllBytes destination) "UTF-8")))))) 288 | 289 | (deftest discard-to-destination 290 | (call-with-temp-dir-path 291 | (fn [tmpdir] 292 | (test-discard-entry-to (.resolve tmpdir "discarded") tmpdir "q1") 293 | (when small-test-fs 294 | (let [dest (Files/createTempFile small-test-fs "discarded-" "" 295 | (into-array FileAttribute []))] 296 | (try 297 | (test-discard-entry-to dest tmpdir "q2") 298 | (finally 299 | (Files/delete dest)))))))) 300 | 301 | (defn fill-filesystem [path] 302 | "Returns truish value if the filesystem containing path is likely full." 303 | (let [append StandardOpenOption/APPEND 304 | buf (byte-array (* 64 1024) (byte \?)) 305 | write-chunks (fn [write-chunk open-opts] 306 | (with-open [out (Files/newOutputStream 307 | path 308 | (into-array OpenOption open-opts))] 309 | (try 310 | (while true (write-chunk out)) 311 | (catch IOException ex true))))] 312 | ;; Write smaller and smaller chunks; finish up with single bytes. 313 | (write-chunks #(.write % buf 0 (* 64 1024)) []) 314 | (write-chunks #(.write % buf 0 1024) [append]) 315 | (write-chunks #(.write % (int \?)) [append]))) 316 | 317 | (deftest full-filesystem-behavior 318 | (when small-test-fs 319 | (let [qdir (.resolve small-test-fs "full-q") 320 | nopedir (.resolve small-test-fs "no-q") 321 | q (stock/create qdir) 322 | balloon (Files/createTempFile small-test-fs "balloon-" "" 323 | (into-array FileAttribute []))] 324 | (try 325 | (let [firehose (future (fill-filesystem balloon)) 326 | result (deref firehose (* 30 1000) nil)] 327 | (is result) 328 | (if-not result 329 | (future-cancel firehose) 330 | (let [free (.getUsableSpace (Files/getFileStore balloon))] 331 | (is (= 0 free)) 332 | (when (zero? free) 333 | (is (thrown? IOException (stock/create nopedir))) 334 | (is (thrown? IOException (store-str q "foo"))))))) 335 | (finally 336 | (Files/delete balloon))) 337 | (let [q (stock/open qdir) 338 | read-entries (stock/reduce q conj ())] 339 | (is (= [] read-entries)))))) 340 | 341 | (def billion 1000000000) 342 | 343 | (deftest uncontended-performance 344 | ;; This also tests random metadata round trips 345 | (call-with-temp-dir-path 346 | (fn [tmpdir] 347 | (let [qdir (.toFile (.resolve tmpdir "queue"))] 348 | (doall 349 | (for [make-meta [nil #(random-path-segment 4)] 350 | batch-size [100 1000] 351 | i (range 3)] 352 | (do 353 | (let [q (stock/create qdir) 354 | ;; Uncontended enqueue 355 | start (System/nanoTime) 356 | items (doall (for [i (range batch-size)] 357 | (let [m (and make-meta (make-meta)) 358 | ent (store-str q (str i) m)] 359 | (when m 360 | (is (= m (stock/entry-meta ent)))) 361 | [m ent]))) 362 | stop (System/nanoTime) 363 | _ (binding [*out* *err*] 364 | (printf "Enqueued %d tiny messages %s metadata at %.2f/s\n" 365 | batch-size 366 | (if make-meta "with" "without") 367 | (double (/ batch-size (/ (- stop start) billion)))) 368 | (flush)) 369 | ;; Uncontended streams 370 | start (System/nanoTime) 371 | _ (is (= (set (map str (range batch-size))) 372 | (set (for [[metadata entry] items] 373 | (slurp-entry q entry))))) 374 | stop (System/nanoTime) 375 | _ (binding [*out* *err*] 376 | (printf "Streamed %d tiny messages %s metadata at %.2f/s\n" 377 | batch-size 378 | (if make-meta "with" "without") 379 | (double (/ batch-size (/ (- stop start) billion)))) 380 | (flush)) 381 | ;; Uncontended discard 382 | start (System/nanoTime) 383 | _ (doseq [[metadata entry] items] 384 | (stock/discard q entry)) 385 | stop (System/nanoTime) 386 | _ (binding [*out* *err*] 387 | (printf "Discarded %d tiny messages %s metadata at %.2f/s\n" 388 | batch-size 389 | (if make-meta "with" "without") 390 | (double (/ batch-size (/ (- stop start) billion)))) 391 | (flush))] 392 | (is (= #{"" "q" "stockpile"} 393 | (set (relative-pathstr-seq (.toPath qdir)))))) 394 | (rm-r (.getAbsolutePath qdir))))))))) 395 | 396 | (deftest contending-enqueue-dequeue-performance 397 | (call-with-temp-dir-path 398 | (fn [tmpdir] 399 | (let [qdir (.toFile (.resolve tmpdir "queue")) 400 | q (stock/create qdir) 401 | batch-size 2000 402 | start (System/nanoTime) 403 | entries (seque (int (max 100 (/ batch-size 10))) 404 | (for [i (range batch-size)] 405 | (store-str q (str i))))] 406 | (doall 407 | (map (fn [i entry] 408 | (is (= (str i) (slurp-entry q entry))) 409 | (stock/discard q entry) 410 | (try 411 | (slurp-entry q entry) 412 | (catch Exception ex 413 | (= {:entry entry :source (entry-path q entry)} 414 | (ex-data ex))))) 415 | (range batch-size) 416 | entries)) 417 | (binding [*out* *err*] 418 | (printf "Enqueued and dequeued %d tiny messages in parallel at %.2f/s\n" 419 | batch-size 420 | (double (/ batch-size 421 | (/ (- (System/nanoTime) start) 422 | billion)))) 423 | (flush)) 424 | (is (= #{"" "q" "stockpile"} 425 | (set (relative-pathstr-seq (.toPath qdir))))))))) 426 | 427 | (deftest simple-race 428 | (call-with-temp-dir-path 429 | (fn [tmpdir] 430 | (let [batch-size 300 431 | qdir (.toFile (.resolve tmpdir "queue")) 432 | q (stock/create qdir) 433 | state (atom {:entries () :victim nil}) 434 | finished? (atom false) 435 | writer (future 436 | (dotimes [i batch-size] 437 | (swap! state update :entries conj 438 | [i (store-str q (str i) (str "meta-" i))]))) 439 | reader (future 440 | (while (not @finished?) 441 | (let [{:keys [victim]} (swap! state 442 | (fn [{[v & r] :entries}] 443 | {:entries r 444 | :victim v})) 445 | [val entry] victim] 446 | (when victim 447 | (is (= (str val) (slurp-entry q entry))) 448 | (swap! state update :entries conj victim))))) 449 | discarder (future 450 | (loop [i 0] 451 | (when (< i batch-size) 452 | (let [{:keys [victim]} (swap! state 453 | (fn [{[v & r] :entries}] 454 | {:entries r 455 | :victim v})) 456 | [val entry] victim] 457 | (if entry 458 | (do 459 | (stock/discard q entry) 460 | (recur (inc i))) 461 | (recur i))))))] 462 | @writer @discarder 463 | (reset! finished? true) 464 | @reader 465 | (is (= #{"" "q" "stockpile"} 466 | (set (relative-pathstr-seq (.toPath qdir))))))))) 467 | --------------------------------------------------------------------------------