├── binding.gyp ├── package.json ├── index.js ├── LICENSE ├── example ├── fast_http.js └── bloom.js ├── README.md ├── run_tests.js └── mmap.cpp /binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "mmap", 5 | "sources": [ "mmap.cpp" ] 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mmap", 3 | "description": "mmap() fds into buffers", 4 | "license": "BSD", 5 | "version": "3.0.0", 6 | "engines":{"node": "5.x.x"}, 7 | "main": "./index.js", 8 | "repository": { "type": "git", "url": "https://github.com/geocar/mmap.git" }, 9 | "scripts": { 10 | "pretest": "npm build .", 11 | "test": "node --expose-gc run_tests.js" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | fs = require("fs"), 2 | mmap = require("./build/Release/mmap"), real_map = mmap.map; 3 | 4 | function mmap_wrapper(size, protection, flags, fd, offset) { 5 | if(typeof fd === "number" || fd instanceof Number) 6 | return real_map(size,protection,flags,fd,offset); 7 | 8 | var writep = (protection & mmap.PROT_WRITE); 9 | if(writep) fs.closeSync(fs.openSync(fd, "a")); 10 | fd = fs.openSync(fd, writep? "r+" : "r"); 11 | var stat = fs.fstatSync(fd); 12 | if(!size) { 13 | size = stat.size; 14 | } else if (writep && size > stat.size) { 15 | fs.ftruncateSync(fd, size); // extend if needed 16 | } 17 | 18 | var buffer = real_map(size, protection, flags, fd, offset); 19 | fs.closeSync(fd); // close-behind 20 | return buffer; 21 | }; 22 | for(var k in mmap) mmap_wrapper[k] = mmap[k]; 23 | mmap_wrapper.map = mmap_wrapper; 24 | 25 | module.exports = mmap_wrapper; 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This node-mmap is based on node-mmap 2 | and as a result, it contains code Copyright (c) 2010 Ben Noordhuis. His license 3 | is below. 4 | 5 | My changes (Geo Carncross) are redistributable under similar terms. 6 | 7 | Copyright (c) 2013 Geo Carncross 8 | 9 | ---- 10 | Copyright (c) 2010 Ben Noordhuis 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy 13 | of this software and associated documentation files (the "Software"), to deal 14 | in the Software without restriction, including without limitation the rights 15 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 | copies of the Software, and to permit persons to whom the Software is 17 | furnished to do so, subject to the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be included in 20 | all copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 28 | THE SOFTWARE. 29 | -------------------------------------------------------------------------------- /example/fast_http.js: -------------------------------------------------------------------------------- 1 | lru = [], 2 | cache = {}, 3 | parseurl = require("url").parse, 4 | fs = require("fs"), 5 | mmap = require("../index.js"); // obv. use require("mmap") if you lift 6 | 7 | require("http").createServer(handler).listen(8080, function() { 8 | console.log("> http ready on http://localhost:8080/"); 9 | }); 10 | 11 | 12 | function handler(req,res) { 13 | var fd, path = parseurl(req.url).pathname; 14 | 15 | try { 16 | fd = fs.openSync(__dirname + path, "r"); 17 | } catch(e) { 18 | console.log(" error = ",e); 19 | res.writeHead(404, "File not found"); 20 | return res.end("404 not found"); 21 | } 22 | var stat = fs.fstatSync(fd); 23 | if(stat.isDirectory()) { 24 | fs.closeSync(fd); 25 | return serve_dir(); 26 | } 27 | 28 | if(cache[path] && cache[path].length == stat.size) return serve_buffer(); 29 | 30 | cache[path] = mmap.map(stat.size, mmap.PROT_READ, mmap.MAP_SHARED, fd); 31 | fs.closeSync(fd); 32 | return serve_buffer(); 33 | 34 | function serve_buffer() { 35 | var buffer = cache[path]; 36 | res.writeHead(200, "OK", {"Content-Type":"text/plain"}); 37 | res.end(buffer); 38 | dolru(); 39 | } 40 | function serve_dir() { 41 | res.writeHead(200, "OK", {"Content-Type":"text/html"}); 42 | res.end(""); 45 | } 46 | function dolru() { 47 | // simple lru handling 48 | lru.unshift(path); 49 | while (lru.length >= 100) { 50 | var lp = lru.pop(); 51 | if(lru.indexOf(lp) === -1) { 52 | delete cache[lp]; 53 | } 54 | } 55 | } 56 | } 57 | function html(x) { 58 | x = "" + x; 59 | return x.replace(/\&/g, "&").replace(/\"/g, """).replace(/ 11 | 12 | n_bytes 13 | The number of bytes to map into memory. 14 | 15 | 16 | protection 17 | Memory protection: either mmap.PROT_NONE or a bitwise OR of mmap.PROT_READ, mmap.PROT_WRITE and mmap.PROT_EXEC. 18 | 19 | 20 | flags 21 | Flags: either mmap.MAP_SHARED or mmap.MAP_PRIVATE. 22 | 23 | 24 | fd 25 | File descriptor. You can also use a file name (string). When you do this, mmap() tries to do the right thing by creating or growing the file to n_bytes if necessary. 26 | 27 | 28 | offset 29 | File offset. Must be either zero or a multiple of mmap.PAGESIZE. 30 | 31 | 32 | 33 | While a finaliser is installed to automatically unmap buffer, you can 34 | force it to unmap immediately with: 35 | 36 | buffer.unmap() 37 | 38 | You can also [msync()](http://pubs.opengroup.org/onlinepubs/9699919799/functions/msync.html) by calling: `buffer.sync([offset, [length, [flags]]])` method: 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 |
offsetThe start address to sync. This argument optional, the default is 0.
lengthThe number of bytes to sync. This argument optional, the default is the entire buffer.
flagsFlags: either mmap.MS_SYNC or mmap.MS_ASYNC optionally bitwise OR with mmap.MS_INVALIDATE. This argument is optional, the default is mmap.MS_SYNC.
54 | 55 | For compatibility, mmap.map() is an alias for mmap() 56 | 57 | ## See Also 58 | 59 | * POSIX 1003.1 [mmap](http://pubs.opengroup.org/onlinepubs/9699919799/functions/mmap.html) and [msync](http://pubs.opengroup.org/onlinepubs/9699919799/functions/msync.html) 60 | * The `example/` directory contains some sample uses of the mmap module 61 | -------------------------------------------------------------------------------- /example/bloom.js: -------------------------------------------------------------------------------- 1 | fs = require("fs"), 2 | mmap = require("../index.js"); // obv. use require("mmap") if you lift 3 | HASH_FUNCTIONS=7; // how many hashes do you want? 4 | 5 | size = 1024*1024; // 1mb 6 | buffer = mmap.map(size, mmap.PROT_READ|mmap.PROT_WRITE, mmap.MAP_SHARED, "/tmp/test_bloom"); 7 | add(buffer, get_offsets("testing", buffer.length)); 8 | console.log("check: ", check(buffer, get_offsets("testing", buffer.length))); 9 | 10 | 11 | /* basic bloom filter tricks */ 12 | function get_offsets(string, buffer_size) { 13 | var a = fnv_1a(string); 14 | var b = fnv_1a_b(a); 15 | var r = []; 16 | var i = -1; 17 | var x = a % buffer_size; 18 | while (++i < HASH_FUNCTIONS) { 19 | r.push(x < 0 ? (x + buffer_size) : x); 20 | x = (x + b) % buffer_size; 21 | } 22 | return r; 23 | } 24 | function add(buffer, offsets) { 25 | var i = -1; 26 | var w = 0; 27 | while (++i < HASH_FUNCTIONS) w += !!(buffer[ offsets[i] ]++); 28 | if (HASH_FUNCTIONS == w) { 29 | while (i-->0) --buffer[offsets[i]]; 30 | return true; 31 | } 32 | return false; 33 | } 34 | function check(buffer, offsets) { 35 | var i = -1; 36 | while (++i < HASH_FUNCTIONS) 37 | if (!buffer[offsets[i]]) 38 | return false; 39 | return true; 40 | } 41 | 42 | /* fowler-noll-vo (fnv) hashing */ 43 | function fnv_1a(v) { 44 | var n = v.length, a = 2166136261, c, d, i = -1; 45 | while (++i < n) { 46 | c = v.charCodeAt(i); 47 | if (d = c & 0xff000000) { 48 | a ^= d >> 24; 49 | a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); 50 | } 51 | if (d = c & 0xff0000) { 52 | a ^= d >> 16; 53 | a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); 54 | } 55 | if (d = c & 0xff00) { 56 | a ^= d >> 8; 57 | a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); 58 | } 59 | a ^= c & 0xff; 60 | a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); 61 | } 62 | /* from http://home.comcast.net/~bretm/hash/6.html */ 63 | a += a << 13; 64 | a ^= a >> 7; 65 | a += a << 3; 66 | a ^= a >> 17; 67 | a += a << 5; 68 | return a & 0xffffffff; 69 | } 70 | 71 | /* one additional iteration of fnv, given a hash */ 72 | function fnv_1a_b(a) { 73 | a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); 74 | a += a << 13; 75 | a ^= a >> 7; 76 | a += a << 3; 77 | a ^= a >> 17; 78 | a += a << 5; 79 | return a & 0xffffffff; 80 | } 81 | 82 | 83 | -------------------------------------------------------------------------------- /run_tests.js: -------------------------------------------------------------------------------- 1 | tempfile=(+new Date()) + "." + process.pid + ".tmp", 2 | assert=require("assert"), fs=require("fs"), mmap = require("./index") 3 | 4 | var t = 128 | (Math.random() * 256); 5 | var b = new Buffer(1); 6 | b[0] = t; 7 | 8 | fs.writeFileSync(tempfile, b); 9 | process.on("exit", function() { 10 | fs.unlink(tempfile); 11 | }); 12 | 13 | (function(b) { 14 | assert.equal(b[0], t, "value check (map read)"); 15 | assert.equal(b[1], 0, "value check (line noise)"); 16 | assert.equal(b[2], 0, "value check (line noise)"); 17 | assert.equal(b.sync(), true, "sync failed"); 18 | 19 | assert.equal(b.length, mmap.PAGESIZE, "map length is incorrect"); 20 | 21 | assert.equal(b.unmap(), true, "unmap failed"); 22 | assert.equal(b.length, 0, "unmap truncated fixed buffer"); 23 | 24 | })( mmap(mmap.PAGESIZE, // note using wrapper (with filename) 25 | mmap.PROT_READ|mmap.PROT_WRITE, 26 | mmap.MAP_SHARED, 27 | tempfile) ); 28 | 29 | fd = fs.openSync(tempfile, "r+"); 30 | (function(b) { 31 | assert.equal(b[0], t, "value check (still there)"); 32 | assert.equal(b[1], 0, "value check (always zero)"); 33 | assert.equal(b[2], 0, "value check (always zero)"); 34 | assert.equal(b.length, mmap.PAGESIZE, "map length is incorrect"); 35 | 36 | b[2] = t; 37 | assert.equal(b.sync(), true, "sync failed"); 38 | 39 | })( mmap(mmap.PAGESIZE, 40 | mmap.PROT_READ|mmap.PROT_WRITE, 41 | mmap.MAP_SHARED, 42 | fd) ); 43 | fs.closeSync(fd); 44 | 45 | 46 | (function(b) { 47 | assert.equal(b[0], t, "value check (still there)"); 48 | assert.equal(b[1], 0, "value check (always zero)"); 49 | assert.equal(b[2], t, "value check (now visible)"); 50 | assert.equal(b.length, mmap.PAGESIZE, "map length is incorrect"); 51 | 52 | assert.equal(b.sync(), true, "sync failed"); 53 | 54 | })( mmap(mmap.PAGESIZE, 55 | mmap.PROT_READ, 56 | mmap.MAP_SHARED, 57 | tempfile) ); 58 | 59 | 60 | fd = fs.openSync(tempfile, "w+"); 61 | fs.truncateSync(fd, mmap.PAGESIZE); 62 | (function(b) { 63 | assert.equal(b[0], 0, "value check (truncated 0)"); 64 | assert.equal(b[1], 0, "value check (always zero)"); 65 | assert.equal(b[2], 0, "value check (always zero)"); 66 | assert.equal(b.length, mmap.PAGESIZE, "map length is incorrect"); 67 | 68 | b[2] = t; 69 | assert.equal(b.sync(), true, "sync failed"); 70 | 71 | })( mmap.map(mmap.PAGESIZE, 72 | mmap.PROT_READ|mmap.PROT_WRITE, 73 | mmap.MAP_SHARED, 74 | fd) ); 75 | 76 | 77 | (function(b) { 78 | fs.closeSync(fd); 79 | 80 | assert.equal(b[0], 0, "value check (truncated 0)"); 81 | assert.equal(b[1], 0, "value check (always zero)"); 82 | assert.equal(b[2], t, "third mapping"); 83 | assert.equal(b.length, mmap.PAGESIZE, "map length is incorrect"); 84 | 85 | })( mmap.map(mmap.PAGESIZE, 86 | mmap.PROT_READ, 87 | mmap.MAP_SHARED, 88 | fd) ); 89 | 90 | fs.writeFileSync(tempfile, b); 91 | 92 | global.gc(); 93 | 94 | (function(b) { 95 | assert.equal(b[0], t, "value check (map read)"); 96 | assert.equal(b[1], 0, "value check (line noise)"); 97 | assert.equal(b[2], 0, "value check (line noise)"); 98 | assert.equal(b.sync(), true, "sync failed"); 99 | 100 | assert.equal(b.length, mmap.PAGESIZE, "map length is incorrect"); 101 | 102 | })( mmap(mmap.PAGESIZE, // note using wrapper (with filename) 103 | mmap.PROT_READ|mmap.PROT_WRITE, 104 | mmap.MAP_SHARED, 105 | tempfile) ); 106 | 107 | global.gc(); 108 | 109 | console.log("> ok"); 110 | -------------------------------------------------------------------------------- /mmap.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | struct hint_wrap { 9 | size_t length; 10 | }; 11 | 12 | 13 | static void Map_finalise(char *data, void*hint_void) 14 | { 15 | struct hint_wrap *h = (struct hint_wrap *)hint_void; 16 | 17 | if(h->length > 0) { 18 | munmap(data, h->length); 19 | } 20 | delete h; 21 | } 22 | 23 | void Sync(const v8::FunctionCallbackInfo& args) 24 | { 25 | auto *isolate = args.GetIsolate(); 26 | auto buffer = args.This()->ToObject(); 27 | char *data = node::Buffer::Data(buffer); 28 | size_t length = node::Buffer::Length(buffer); 29 | 30 | // First optional argument: offset 31 | if (args.Length() > 0) { 32 | const size_t offset = args[0]->ToInteger()->Value(); 33 | if(length <= offset) return; 34 | 35 | data += offset; 36 | length -= offset; 37 | } 38 | 39 | // Second optional argument: length 40 | if (args.Length() > 1) { 41 | const size_t range = args[1]->ToInteger()->Value(); 42 | if(range < length) length = range; 43 | } 44 | 45 | // Third optional argument: flags 46 | int flags; 47 | if (args.Length() > 2) { 48 | flags = args[2]->ToInteger()->Value(); 49 | } else { 50 | flags = MS_SYNC; 51 | } 52 | 53 | args.GetReturnValue().Set((0 == msync(data, length, flags)) ? v8::True(isolate) : v8::False(isolate)); 54 | } 55 | 56 | void Unmap(const v8::FunctionCallbackInfo& args) 57 | { 58 | auto *isolate = args.GetIsolate(); 59 | auto buffer = args.This()->ToObject(); 60 | char *data = node::Buffer::Data(buffer); 61 | 62 | struct hint_wrap *d = (struct hint_wrap *)v8::External::Cast(*buffer->GetHiddenValue(v8::String::NewFromUtf8(isolate,"mmap_dptr")))->Value(); 63 | 64 | bool ok = true; 65 | 66 | if(d->length > 0 && -1 == munmap(data, d->length)) { 67 | ok = false; 68 | } else { 69 | d->length = 0; 70 | (void)buffer->CreateDataProperty(isolate->GetCurrentContext(), 71 | v8::String::NewFromUtf8(isolate, "length"), 72 | v8::Number::New(isolate, 0)); 73 | } 74 | 75 | args.GetReturnValue().Set(ok? v8::True(isolate): v8::False(isolate)); 76 | } 77 | 78 | void Map(const v8::FunctionCallbackInfo& args) 79 | { 80 | auto *isolate = args.GetIsolate(); 81 | 82 | if (args.Length() <= 3) 83 | { 84 | isolate->ThrowException( 85 | v8::Exception::Error( 86 | v8::String::NewFromUtf8(isolate, "mmap() takes 4 arguments: size, protection, flags, fd and offset."))); 87 | return; 88 | } 89 | 90 | const size_t length = args[0]->ToInteger()->Value(); 91 | const int protection = args[1]->ToInteger()->Value(); 92 | const int flags = args[2]->ToInteger()->Value(); 93 | const int fd = args[3]->ToInteger()->Value(); 94 | const off_t offset = args[4]->ToInteger()->Value(); 95 | 96 | char* data = (char *) mmap(0, length, protection, flags, fd, offset); 97 | 98 | if(data == MAP_FAILED) 99 | { 100 | isolate->ThrowException(node::ErrnoException(isolate, errno, "mmap", "")); 101 | return; 102 | } 103 | 104 | struct hint_wrap *d = new hint_wrap; 105 | d->length = length; 106 | 107 | auto buffer = node::Buffer::New(isolate, data, length, Map_finalise, (void*)d).ToLocalChecked(); 108 | auto buffer_object = buffer->ToObject(); 109 | 110 | buffer_object->Set(v8::String::NewFromUtf8(isolate, "unmap"), v8::FunctionTemplate::New(isolate, Unmap)->GetFunction()); 111 | buffer_object->Set(v8::String::NewFromUtf8(isolate, "sync"), v8::FunctionTemplate::New(isolate, Sync)->GetFunction()); 112 | buffer_object->SetHiddenValue(v8::String::NewFromUtf8(isolate,"mmap_dptr"), v8::External::New(isolate, (void*)d)); 113 | 114 | args.GetReturnValue().Set(buffer); 115 | } 116 | 117 | 118 | static void RegisterModule(v8::Local exports) 119 | { 120 | const int PAGESIZE = sysconf(_SC_PAGESIZE); 121 | 122 | NODE_SET_METHOD(exports, "map", Map); 123 | NODE_DEFINE_CONSTANT(exports, PROT_READ); 124 | NODE_DEFINE_CONSTANT(exports, PROT_WRITE); 125 | NODE_DEFINE_CONSTANT(exports, PROT_EXEC); 126 | NODE_DEFINE_CONSTANT(exports, PROT_NONE); 127 | NODE_DEFINE_CONSTANT(exports, MAP_SHARED); 128 | NODE_DEFINE_CONSTANT(exports, MAP_PRIVATE); 129 | NODE_DEFINE_CONSTANT(exports, PAGESIZE); 130 | NODE_DEFINE_CONSTANT(exports, MS_ASYNC); 131 | NODE_DEFINE_CONSTANT(exports, MS_SYNC); 132 | NODE_DEFINE_CONSTANT(exports, MS_INVALIDATE); 133 | } 134 | 135 | NODE_MODULE(mmap, RegisterModule); 136 | --------------------------------------------------------------------------------