├── .gitignore ├── README.md ├── doc └── intro.md ├── project.clj ├── src └── clj_mmap.clj └── test └── clj_mmap └── core_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | *jar 3 | /lib/ 4 | /classes/ 5 | /targets/ 6 | .lein-deps-sum 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Available via [clojars](https://clojars.org/clj-mmap) 2 | Current stable version: [clj-mmap "1.1.2"] 3 | 4 | 5 | # clj-mmap 6 | 7 | A Clojure library designed to allow you to easily mmap files via Java's NIO, and to handle files larger than 2GB. 8 | 9 | 10 | ## Usage 11 | ```clojure 12 | (with-open [mapped-file (clj-mmap/get-mmap "/tmp/big_file.txt")] 13 | (let [some-bytes (clj-mmap/get-bytes mapped-file 0 30)] 14 | (println (str "First 30 bytes of file, '" (String. some-bytes "UTF-8") "'")))) 15 | ``` 16 | 17 | 18 | ## Artifacts 19 | 20 | clj-mmap artifacts are [released to Clojars](https://clojars.org/clj-mmap). 21 | 22 | If you are using Maven, add the following repository definition to your `pom.xml`: 23 | 24 | ``` xml 25 | 26 | clojars 27 | http://clojars.org/repo 28 | 29 | ``` 30 | 31 | ### The Most Recent Release 32 | 33 | With Leiningen: 34 | 35 | [clj-mmap "1.1.2"] 36 | 37 | 38 | With Maven: 39 | 40 | 41 | clj-mmap 42 | clj-mmap 43 | 1.1.2 44 | 45 | 46 | 47 | ## License 48 | 49 | MIT 50 | http://opensource.org/licenses/MIT 51 | 52 | I'd also like to thank my employer, Gracenote, for allowing me to create this open source port. 53 | 54 | Copyright (C) 2012-2013 Alan Busby -------------------------------------------------------------------------------- /doc/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to clj-mmap 2 | 3 | TODO: write [great documentation](http://jacobian.org/writing/great-documentation/what-to-write/) 4 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject clj-mmap "1.1.2" 2 | :description "A wrapper over java.nio's mmap() implementation to ease use, and enable mmap'ing files larger than 2GB." 3 | :url "https://github.com/thebusby/clj-mmap" 4 | :license {:name "MIT" 5 | :url "http://opensource.org/licenses/MIT"} 6 | :dependencies [[org.clojure/clojure "1.5.1"]]) 7 | -------------------------------------------------------------------------------- /src/clj_mmap.clj: -------------------------------------------------------------------------------- 1 | (ns clj-mmap) 2 | 3 | (set! *warn-on-reflection* true) 4 | 5 | (def ^:private bytes-per-map 6 | "The number of bytes a single MappedByteBuffer will store" 7 | java.lang.Integer/MAX_VALUE) 8 | 9 | (definterface ISize 10 | (^long size [])) 11 | 12 | (deftype Mmap [^java.io.RandomAccessFile fis ^java.nio.channels.FileChannel fc maps] 13 | ISize 14 | (size [this] (.size fc)) 15 | 16 | clojure.lang.Indexed 17 | (nth [this i] (get maps i)) 18 | (nth [this i not-found] (get maps i not-found)) 19 | 20 | clojure.lang.Seqable 21 | (seq [this] (seq maps)) 22 | 23 | java.io.Closeable 24 | (close 25 | [this] 26 | (do 27 | (.close fc) 28 | (.close fis)))) 29 | 30 | (def ^:private map-modes 31 | {:private java.nio.channels.FileChannel$MapMode/PRIVATE 32 | :read-only java.nio.channels.FileChannel$MapMode/READ_ONLY 33 | :read-write java.nio.channels.FileChannel$MapMode/READ_WRITE}) 34 | 35 | (def ^:private map-perms 36 | {:private "r" 37 | :read-only "r" 38 | :read-write "rw"}) 39 | 40 | (defn get-mmap 41 | "Provided a filename, mmap the entire file, and return an opaque type to allow further access. 42 | Remember to use with-open, or to call .close, to clean up memory and open file descriptors." 43 | ([^String filename] (get-mmap filename :read-only)) 44 | ([^String filename map-mode] 45 | (let [fis (java.io.RandomAccessFile. filename (map-perms map-mode)) 46 | fc (.getChannel fis) 47 | size (.size fc) 48 | mmap (fn [pos n] (.map fc (map-modes map-mode) pos n))] 49 | (Mmap. fis fc (mapv #(mmap % (min (- size %) 50 | bytes-per-map)) 51 | (range 0 size bytes-per-map)))))) 52 | 53 | (defn get-bytes ^bytes [mmap pos n] 54 | "Retrieve n bytes from mmap, at byte position pos." 55 | (let [get-chunk #(nth mmap (int (/ % bytes-per-map))) 56 | end (+ pos n) 57 | chunk-term (-> pos 58 | (/ bytes-per-map) 59 | int 60 | inc 61 | (* bytes-per-map)) 62 | read-size (- (min end chunk-term) ;; bytes to read in first chunk 63 | pos) 64 | start-chunk ^java.nio.MappedByteBuffer (get-chunk pos) 65 | end-chunk ^java.nio.MappedByteBuffer (get-chunk end) 66 | buf (byte-array n)] 67 | 68 | (locking start-chunk 69 | (.position start-chunk (mod pos bytes-per-map)) 70 | (.get start-chunk buf 0 read-size)) 71 | 72 | ;; Handle reads that span MappedByteBuffers 73 | (if (not= start-chunk end-chunk) 74 | (locking end-chunk 75 | (.position end-chunk 0) 76 | (.get end-chunk buf read-size (- n read-size)))) 77 | 78 | buf)) 79 | 80 | (defn put-bytes 81 | "Write n bytes from buf into mmap, at byte position pos. 82 | If n isn't provided, the size of the buffer provided is used." 83 | ([mmap ^bytes buf pos] (put-bytes mmap buf pos (alength buf))) 84 | ([mmap ^bytes buf pos n] 85 | (let [get-chunk #(nth mmap (int (/ % bytes-per-map))) 86 | end (+ pos n) 87 | chunk-term (-> pos 88 | (/ bytes-per-map) 89 | int 90 | inc 91 | (* bytes-per-map)) 92 | write-size (- (min end chunk-term) 93 | pos) 94 | start-chunk ^java.nio.MappedByteBuffer (get-chunk pos) 95 | end-chunk ^java.nio.MappedByteBuffer (get-chunk end)] 96 | 97 | (locking start-chunk 98 | (.position start-chunk (mod pos bytes-per-map)) 99 | (.put start-chunk buf 0 write-size)) 100 | 101 | ;; Handle writes that span MappedByteBuffers 102 | (if (not= start-chunk end-chunk) 103 | (locking end-chunk 104 | (.position end-chunk 0) 105 | (.put end-chunk buf write-size (- n write-size)))) 106 | 107 | nil))) 108 | 109 | (defn loaded? [mmap] 110 | "Returns true if it is likely that the buffer's contents reside in physical memory." 111 | (every? (fn [^java.nio.MappedByteBuffer buf] 112 | (.isLoaded buf)) 113 | mmap)) 114 | -------------------------------------------------------------------------------- /test/clj_mmap/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-mmap.core-test 2 | (:require [clojure.test :refer :all] 3 | [clj-mmap.core :refer :all])) 4 | 5 | (deftest a-test 6 | (testing "FIXME, no test cases provided!" 7 | (is (= 1 1)))) 8 | --------------------------------------------------------------------------------