├── .gitignore ├── CHANGELOG.md ├── project.clj ├── LICENSE ├── test └── co │ └── deps │ └── ring_etag_middleware_test.clj ├── src └── co │ └── deps │ └── ring_etag_middleware.clj └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 3 | 4 | ## [Unreleased] 5 | 6 | ## [0.2.1] - 2019-10-17 7 | 8 | ### Added 9 | 10 | * Three-arity async ring handler - [#2](https://github.com/deps-app/ring-etag-middleware/pull/2) [@ianbishop](https://github.com/ianbishop) 11 | 12 | ## [0.2.0] - 2018-03-20 13 | 14 | Initial release 15 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject co.deps/ring-etag-middleware "0.2.2-SNAPSHOT" 2 | :description "Ring middleware to set an ETag on responses" 3 | :url "https://github.com/deps-app/ring-etag-middleware" 4 | :license {:name "MIT" 5 | :url "https://opensource.org/licenses/MIT"} 6 | :global-vars {*warn-on-reflection* true} 7 | :deploy-repositories {"releases" :clojars 8 | "snapshots" :clojars} 9 | :dependencies [[org.clojure/clojure "1.8.0"] 10 | [ring/ring-core "1.6.3"]]) 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Daniel Compton 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /test/co/deps/ring_etag_middleware_test.clj: -------------------------------------------------------------------------------- 1 | (ns co.deps.ring-etag-middleware-test 2 | (:require [clojure.test :refer :all] 3 | [co.deps.ring-etag-middleware :refer :all] 4 | [ring.util.response :refer [get-header]]) 5 | (:import (java.nio.file Files) 6 | (java.nio.file.attribute FileAttribute) 7 | (java.io File))) 8 | 9 | (def ^{:dynamic true :tag File} *temp-file* nil) 10 | 11 | (defn ^File create-temp-file [] 12 | (let [path (Files/createTempFile "temp-figwheel-test-file" ".js" (into-array FileAttribute [])) 13 | file (.toFile path)] 14 | (.deleteOnExit file) 15 | file)) 16 | 17 | (defn- constant-handler [body] 18 | (constantly 19 | {:status 200 20 | :headers {} 21 | :body body})) 22 | 23 | (def js-string "// Compiled by ClojureScript 1.9.908 {} 24 | goog.provide('myapp.core'); 25 | goog.require('cljs.core'); 26 | ") 27 | 28 | (use-fixtures 29 | :each 30 | (fn [f] 31 | (let [file (create-temp-file)] 32 | (spit file js-string) 33 | (.deleteOnExit file) 34 | (binding [*temp-file* file] 35 | (f))))) 36 | 37 | (deftest add-file-etag-test 38 | (is (= (get-header (add-file-etag {:body *temp-file*} false) 39 | "ETag") 40 | "3677941581"))) 41 | 42 | (deftest add-file-etag-extended-attributes-test 43 | (when (supports-extended-attributes? (.toPath *temp-file*)) 44 | (let [mod-date (.lastModified *temp-file*)] 45 | (is (nil? (get-attribute (.toPath *temp-file*) checksum-attribute-name))) 46 | (is (= (get-header (add-file-etag {:body *temp-file*} true) "ETag") 47 | "3677941581")) 48 | (is (= (get-attribute (.toPath *temp-file*) checksum-attribute-name) 49 | "3677941581")) 50 | (testing "adding attributes to file doesn't update last modified date" 51 | (is (= mod-date (.lastModified *temp-file*)))) 52 | (is (= (get-header (add-file-etag {:body *temp-file*} true) "ETag") 53 | "3677941581"))))) 54 | 55 | (deftest wrap-file-etag-test 56 | (testing "file response" 57 | (is (= (dissoc ((wrap-file-etag (constant-handler *temp-file*)) {:request-method :get, :uri "/"}) 58 | :body) 59 | {:headers {"ETag" "3677941581"} 60 | :status 200}))) 61 | (testing "string response has no etag" 62 | (is (= ((wrap-file-etag (constant-handler "test")) {:request-method :get, :uri "/"}) 63 | {:status 200 64 | :headers {} 65 | :body "test"}))) 66 | (testing "map response has no etag" 67 | (is (= ((wrap-file-etag (constant-handler {:test 1 :time 2})) {:request-method :get, :uri "/"}) 68 | {:status 200 69 | :headers {} 70 | :body {:test 1 :time 2}})))) 71 | 72 | #_(deftest speed-test 73 | (let [temp-file (create-temp-file) 74 | temp-file-ext (create-temp-file)] 75 | (println "Calculate checksum every time") 76 | (dotimes [_ 10] 77 | (time 78 | (dotimes [_ 1000] 79 | (add-file-etag {:body temp-file} false)))) 80 | (println "Calculate checksum once and store it as an extended attribute") 81 | (when (supports-extended-attributes? (.toPath temp-file-ext)) 82 | (dotimes [_ 10] 83 | (time 84 | (dotimes [_ 1000] 85 | (add-file-etag {:body temp-file-ext} true))))))) 86 | -------------------------------------------------------------------------------- /src/co/deps/ring_etag_middleware.clj: -------------------------------------------------------------------------------- 1 | (ns co.deps.ring-etag-middleware 2 | (:require [clojure.java.io :as io] 3 | [ring.util.response :as response]) 4 | (:import (java.util.zip CRC32) 5 | (java.io File) 6 | (java.nio.file Path Files LinkOption FileSystemException) 7 | (java.nio.file.attribute UserDefinedFileAttributeView) 8 | (java.nio ByteBuffer) 9 | (java.nio.charset Charset))) 10 | 11 | (defn checksum-file 12 | "Calculate a CRC32 checksum for a File." 13 | ;; Copied from code generated by Pandect 14 | ;; https://github.com/xsc/pandect 15 | [^File file] 16 | (with-open [is (io/input-stream file)] 17 | (let [buffer-size (int 2048) 18 | ba (byte-array buffer-size) 19 | crc-32 (new CRC32)] 20 | (loop [] 21 | (let [num-bytes-read (.read is ba 0 buffer-size)] 22 | (when-not (= num-bytes-read -1) 23 | (.update crc-32 ba 0 num-bytes-read) 24 | (recur)))) 25 | (.getValue crc-32)))) 26 | 27 | (defn ^UserDefinedFileAttributeView get-user-defined-attribute-view [path] 28 | (Files/getFileAttributeView 29 | path 30 | UserDefinedFileAttributeView 31 | (into-array LinkOption []))) 32 | 33 | (def checksum-attribute-name "user.ring-etag-middleware.crc32-checksum") 34 | 35 | (defn get-attribute [path attribute] 36 | (try 37 | (let [view (get-user-defined-attribute-view path) 38 | name attribute 39 | size (.size view name) 40 | attr-buf (ByteBuffer/allocate size)] 41 | (.read view name attr-buf) 42 | (.flip attr-buf) 43 | (str (.decode (Charset/defaultCharset) attr-buf))) 44 | (catch FileSystemException e 45 | nil))) 46 | 47 | (defn set-attribute [path attribute ^String value] 48 | (let [view (get-user-defined-attribute-view path)] 49 | (.write view attribute (.encode (Charset/defaultCharset) value)))) 50 | 51 | ;; Public API 52 | 53 | (defn supports-extended-attributes? 54 | "The JDK doesn't support UserDefinedFileAttributes (a.k.a. extended attributes) 55 | on all platforms. 56 | 57 | Notably, HFS+ and APFS on macOS do not support extended attributes on JDK 16 and 58 | below. Support was added in JDK 17: https://bugs.openjdk.java.net/browse/JDK-8030048." 59 | [^Path path] 60 | (.supportsFileAttributeView 61 | (Files/getFileStore path) 62 | ^Class UserDefinedFileAttributeView)) 63 | 64 | (defn add-file-etag 65 | [response extended-attributes?] 66 | (let [file (:body response)] 67 | (if (instance? File file) 68 | (let [path (.toPath ^File file)] 69 | (if extended-attributes? 70 | (if-let [checksum (get-attribute path checksum-attribute-name)] 71 | (response/header response "ETag" checksum) 72 | (let [checksum (checksum-file file)] 73 | (set-attribute path checksum-attribute-name (str checksum)) 74 | (response/header response "ETag" checksum))) 75 | (response/header response "ETag" (checksum-file file)))) 76 | response))) 77 | 78 | (defn wrap-file-etag 79 | "Calculates an ETag for a Ring response which contains a File as the body. 80 | 81 | If extended-attributes? is true, then the File is first checked for a 82 | checksum in it's extended attributes, if it doesn't exist then it is 83 | calculated and added to the file, and returned in the ETag. This is 84 | much faster than calculating the checksum each time (which is already 85 | fast), but isn't supported on all platforms, notably macOS. 86 | 87 | If you wish to store the checksum in extended attributes, it is 88 | recommended that you first check if the Path that you are wanting 89 | to serve files from supports it. You can use the provided 90 | supports-extended-attributes? function for this." 91 | ([handler] 92 | (wrap-file-etag handler {})) 93 | ([handler {:keys [extended-attributes?] :as options}] 94 | (fn 95 | ([req] 96 | (add-file-etag (handler req) extended-attributes?)) 97 | ([req respond raise] 98 | (handler req 99 | #(respond (add-file-etag % extended-attributes?)) 100 | raise))))) 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ring-etag-middleware 2 | 3 | [![Clojars Project](https://img.shields.io/clojars/v/co.deps/ring-etag-middleware.svg)](https://clojars.org/co.deps/ring-etag-middleware) [![CircleCI](https://circleci.com/gh/danielcompton/ring-ip-whitelist.svg?style=svg)](https://circleci.com/gh/danielcompton/ring-ip-whitelist) [![Dependencies Status](https://versions.deps.co/deps-app/ring-etag-middleware/status.svg)](https://versions.deps.co/deps-app/ring-etag-middleware) 4 | 5 | Calculates [ETags](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) for [Ring](https://github.com/ring-clojure/ring) File responses using a CRC32 checksum. 6 | 7 | ## Usage 8 | 9 | Add this to your `project.clj` or `build.boot`: 10 | 11 | ``` 12 | [co.deps/ring-etag-middleware "0.2.1"] 13 | ``` 14 | 15 | Require the namespace and add `wrap-file-etag` to your middleware stack: 16 | 17 | ```clojure 18 | (ns my-app.core 19 | (:require [co.deps.ring-etag-middleware :as etag])) 20 | 21 | (-> handler 22 | (etag/wrap-file-etag)) 23 | ``` 24 | 25 | ### Returning 304 Not Modified responses 26 | 27 | This middleware only calculates checksums, it doesn't make any decisions about the status code returned to the client. If the User Agent has provided an Etag in an [If-None-Match](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) header that matches what is calculated by the server, then you probably want to return a [304 Not Modified](https://httpstatuses.com/304) response. I recommend using the middleware built-in to Ring, [`wrap-not-modified`](http://ring-clojure.github.io/ring/ring.middleware.not-modified.html). 28 | 29 | ```clojure 30 | (ns my-app.core 31 | (:require [co.deps.ring-etag-middleware :as etag] 32 | [ring.middleware.not-modified :as not-modified])) 33 | 34 | (-> handler 35 | (etag/wrap-file-etag) 36 | (not-modified/wrap-not-modified)) 37 | ``` 38 | 39 | For a more complete example, you can see the [middleware configuration](https://github.com/bhauman/lein-figwheel/blob/v0.5.17/sidecar/src/figwheel_sidecar/components/figwheel_server.clj#L261-L263) that Figwheel uses. 40 | 41 | ## Caching checksum calculations 42 | 43 | Once a checksum for a file has been calculated once, it is unnecessary to calculate it again. If the files you are serving are immutable, then it would be possible to pre-calculate the checksum once and store the checksum in a local atom. However if you are working in an environment where the files being served may change (say a ClojureScript compiler output directory), then you cannot store the checksum separately from the file (either in-memory or on-disk), as you don't have a 100% reliable method for detecting when to recalculate the checksum (without running a file watcher, which introduces its own problems). 44 | 45 | Instead, ring-etag-middleware provides a way to store checksums in the [extended attributes](https://en.wikipedia.org/wiki/Extended_file_attributes) of the files being served. If this option is enabled, the middleware will check if the `java.io.File` in the Ring response has a checksum already calculated. If so it will return it as the ETag; if not it will calculate the checksum and store it as an extended attribute on the `File`. The JDK doesn't support this on all platforms that have support for extended attributes (notably JDK 16 and below don't support extended attributes on [macOS](https://bugs.openjdk.java.net/browse/JDK-8030048)), so it is recommended to check for support with the provided `supports-extended-attributes?` function. 46 | 47 | ```clojure 48 | (ns my-app.web 49 | (:require [co.deps.ring-etag-middleware :as etag] 50 | [clojure.java.io :as io])) 51 | 52 | (def file-path 53 | (.toPath (io/file "./public/files") 54 | ;; or (Paths/get "./public/files" (into-array String [])) 55 | )) 56 | 57 | (-> handler 58 | (etag/wrap-file-etag 59 | {:extended-attributes? 60 | (etag/supports-extended-attributes? file-path)})) 61 | ``` 62 | 63 | ## Checksums or hashes? 64 | 65 | [Checksums](https://en.wikipedia.org/wiki/Checksum) are faster to calculate than [cryptographic hash functions](https://en.wikipedia.org/wiki/Cryptographic_hash_function) like MD5 or SHA1. An ETag doesn't need any of the cryptographic properties that hash functions provide, so using a checksum is a better choice. Pandect has some [benchmarks](https://github.com/xsc/pandect#benchmark-results) showing the speed differences between checksums and hashes. 66 | 67 | We use CRC32 over Adler32 because it has a [lower risk](https://www.leviathansecurity.com/blog/analysis-of-adler32) of collisions at the cost of being slightly slower to calculate (10-20%). If you are at all concerned about performance, you should enable storing checksums in file extended attributes. 68 | 69 | **Order of magnitude performance notes** 70 | 71 | All benchmarks taken on an unloaded M1 MacBook Air. Note that these are best possible case numbers as the benchmark ensures all files are in memory. 72 | 73 | * Reading an extended attribute at various file sizes: ~20 µs 74 | * CRC32 checksum of 50 kB file: ~30 µs 75 | * CRC32 checksum of 2.7 MB file: ~500 µs 76 | * CRC32 checksum of 13 MB file: ~3 ms 77 | 78 | ## Serving ClojureScript files 79 | 80 | I've written a [blog post](https://danielcompton.net/2018/03/21/how-to-serve-clojurescript) detailing how this is used to serve ClojureScript files and avoid caching inconsistencies. 81 | 82 | ## License 83 | 84 | Copyright © 2018 Daniel Compton 85 | 86 | Distributed under the MIT license. 87 | --------------------------------------------------------------------------------