├── .busted ├── .gitignore ├── .travis.yml ├── AUTHORS ├── LICENSE ├── README.md ├── appveyor.yml ├── makefile ├── rockspecs ├── mmapfile-1-1.rockspec ├── mmapfile-2-1.rockspec ├── mmapfile-3-1.rockspec ├── mmapfile-4-1.rockspec ├── mmapfile-scm-1.rockspec ├── mmapfile-scm-2.rockspec ├── mmapfile-scm-3.rockspec └── mmapfile-scm-4.rockspec ├── spec └── mmapfile_spec.lua └── src-lua ├── config.ld ├── mmapfile.lua └── mmapfile ├── string_list.lua ├── unix.lua └── windows.lua /.busted: -------------------------------------------------------------------------------- 1 | return 2 | { 3 | default = 4 | { 5 | verbose = true, 6 | lpath = "./src-lua/?.lua", 7 | ROOT = "./spec", 8 | output = "utfTerminal", 9 | }, 10 | travis = 11 | { 12 | lpath = "./src-lua/?.lua", 13 | ROOT = "./spec", 14 | output = "TAP", 15 | } 16 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_store 2 | doc 3 | test* 4 | luacov.report.out 5 | luacov.stats.out 6 | mmapfile*.src.rock -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | 4 | env: 5 | - LUA='luajit=2.0' 6 | - LUA='luajit=2.1' 7 | 8 | before_install: 9 | - pip install hererocks 10 | - hererocks lua_install -r^ --$LUA 11 | - export PATH=$PATH:$PWD/lua_install/bin 12 | 13 | 14 | install: 15 | - luarocks install luacheck 16 | - luarocks install busted 17 | - luarocks install luacov 18 | - luarocks install luacov-coveralls 19 | - luarocks make rockspecs/mmapfile-scm-2.rockspec 20 | 21 | script: 22 | - luacheck src-lua 23 | - luacheck --std max+busted spec 24 | - busted --run=travis --coverage . 25 | 26 | after_success: 27 | - luacov-coveralls -i src%-lua 28 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Leyland, Geoff -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2016 Incremental IP Limited 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lua-mmapfile - A simple interface to mmap 2 | 3 | [![Unix Build Status](https://travis-ci.org/geoffleyland/lua-mmapfile.svg?branch=master)](https://travis-ci.org/geoffleyland/lua-mmapfile) 4 | [![Windows Build status](https://ci.appveyor.com/api/projects/status/k553fj0isidhmkoc?svg=true)](https://ci.appveyor.com/project/geoffleyland/lua-mmapfile) 5 | [![Coverage Status](https://coveralls.io/repos/github/geoffleyland/lua-mmapfile/badge.svg?branch=master)](https://coveralls.io/github/geoffleyland/lua-mmapfile?branch=master) 6 | 7 | ## 1. What? 8 | 9 | mmapfile uses `mmap` on unix and `MapViewOfFile` on Windows to provide a way 10 | of quickly storing and loading data that's already in some kind of in-memory 11 | binary format. 12 | 13 | `create` creates a new file and maps new memory to that file. You can 14 | then write to the memory to write to the file. 15 | 16 | `open` opens an existing file and maps its contents to memory, returning 17 | a pointer to the memory and the length of the file. 18 | 19 | `close` syncs memory to the file, closes the file, and deletes the 20 | mapping between the memory and the file. 21 | 22 | `malloc` maps anonymous memory (not mapped to a file), and so acts like 23 | malloc. The advantage is that we can get memory above 4G, but the BIG 24 | disadvantage is that mmap is not a high-perfomance allocator. Only use this 25 | infrequently for big blocks of memory. It's a nasty hack. 26 | 27 | `free` frees memory mmapped by `malloc` 28 | 29 | The "gc" variants of `create`, `open` and `malloc` (`gccreate`, `gcopen` and 30 | `gcmalloc`) set up a garbage collection callback for the pointer so that the 31 | file is correctly closed when the pointer is no longer referenced. Not 32 | appropriate if you might be storing the pointer in C, referencing it from 33 | unmanaged memory, or casting it to another type! 34 | 35 | All memory is mapped above 4G to try to keep away from the memory space 36 | LuaJIT uses. 37 | 38 | 39 | ## 2. How? 40 | 41 | local ffi = require"ffi" 42 | 43 | ffi.cdef"struct test { int a; double b; };" 44 | 45 | local mmapfile = require"mmapfile" 46 | 47 | local ptr1 = mmapfile.gccreate("mmapfile-test", 1, "struct test") 48 | ptr1.a = 1 49 | ptr1.b = 1.5 50 | ptr1 = nil 51 | collectgarbage() 52 | 53 | local ptr2, size = mmapfile.gcopen("mmapfile-test", "struct test") 54 | assert(size == 1) 55 | assert(ptr2.a == 1) 56 | assert(ptr2.b == 1.5) 57 | 58 | For more details `make doc` or `ldoc lua --all`. 59 | 60 | 61 | ## 3. Requirements 62 | 63 | + [LuaJIT](http://luajit.org) and 64 | + [ljsyscall](https://github.com/justincormack/ljsyscall) on Unix 65 | 66 | 67 | ## 4. Issues 68 | 69 | + Should probably have an option for directly mapping existing memory, but 70 | I don't know enough about page boundaries. 71 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | matrix: 3 | - LUA: "luajit 2.0" 4 | - LUA: "luajit 2.1" 5 | 6 | before_build: 7 | - set PATH=C:\Python27\Scripts;%PATH% # Add directory containing 'pip' to PATH 8 | - pip install hererocks 9 | - hererocks env --%LUA% -rlatest 10 | - call env\bin\activate 11 | - luarocks install busted 12 | 13 | build_script: 14 | - luarocks make rockspecs/mmapfile-scm-2.rockspec 15 | 16 | test_script: 17 | - busted . -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | LUA= $(shell echo `which lua`) 2 | LUA_BINDIR= $(shell echo `dirname $(LUA)`) 3 | LUA_PREFIX= $(shell echo `dirname $(LUA_BINDIR)`) 4 | LUA_VERSION = $(shell echo `lua -v 2>&1 | cut -d " " -f 2 | cut -b 1-3`) 5 | LUA_SHAREDIR=$(LUA_PREFIX)/share/lua/$(LUA_VERSION) 6 | 7 | default: 8 | @echo "Nothing to build. Try 'make install'." 9 | 10 | install: 11 | cp lua/mmapfile.lua $(LUA_SHAREDIR) 12 | 13 | doc: lua/mmapfile.lua lua/config.ld 14 | ldoc lua --all 15 | 16 | test: 17 | cd lua && luajit test/test.lua -------------------------------------------------------------------------------- /rockspecs/mmapfile-1-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "1-1" 3 | source = 4 | { 5 | url = "git://github.com/geoffleyland/lua-mmapfile.git", 6 | branch = "master", 7 | tag = "v1", 8 | } 9 | description = 10 | { 11 | summary = "Simple memory-mapped files", 12 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 13 | license = "MIT/X11", 14 | maintainer = "Geoff Leyland " 15 | } 16 | dependencies = 17 | { 18 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 19 | 'ljsyscall >= 0.9', 20 | } 21 | build = 22 | { 23 | type = "builtin", 24 | modules = 25 | { 26 | mmapfile = "lua/mmapfile.lua", 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /rockspecs/mmapfile-2-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "2-1" 3 | source = 4 | { 5 | url = "git://github.com/geoffleyland/lua-mmapfile.git", 6 | branch = "master", 7 | tag = "v2", 8 | } 9 | description = 10 | { 11 | summary = "Simple memory-mapped files", 12 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 13 | license = "MIT/X11", 14 | maintainer = "Geoff Leyland " 15 | } 16 | dependencies = 17 | { 18 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 19 | platforms = 20 | { 21 | linux = { 'ljsyscall >= 0.9' }, 22 | macosx = { 'ljsyscall >= 0.9' }, 23 | } 24 | } 25 | build = 26 | { 27 | type = "builtin", 28 | modules = 29 | { 30 | mmapfile = "src-lua/mmapfile.lua", 31 | ["mmapfile.unix"] = "src-lua/mmapfile/unix.lua", 32 | ["mmapfile.windows"] = "src-lua/mmapfile/windows.lua", 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /rockspecs/mmapfile-3-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "3-1" 3 | source = 4 | { 5 | url = "git://github.com/geoffleyland/lua-mmapfile.git", 6 | branch = "master", 7 | tag = "v3", 8 | } 9 | description = 10 | { 11 | summary = "Simple memory-mapped files", 12 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 13 | license = "MIT/X11", 14 | maintainer = "Geoff Leyland " 15 | } 16 | dependencies = 17 | { 18 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 19 | platforms = 20 | { 21 | linux = { 'ljsyscall >= 0.9' }, 22 | macosx = { 'ljsyscall >= 0.9' }, 23 | } 24 | } 25 | build = 26 | { 27 | type = "builtin", 28 | modules = 29 | { 30 | mmapfile = "src-lua/mmapfile.lua", 31 | ["mmapfile.unix"] = "src-lua/mmapfile/unix.lua", 32 | ["mmapfile.windows"] = "src-lua/mmapfile/windows.lua", 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /rockspecs/mmapfile-4-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "4-1" 3 | source = 4 | { 5 | url = "git://github.com/geoffleyland/lua-mmapfile.git", 6 | branch = "master", 7 | tag = "v3", 8 | } 9 | description = 10 | { 11 | summary = "Simple memory-mapped files", 12 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 13 | license = "MIT/X11", 14 | maintainer = "Geoff Leyland " 15 | } 16 | dependencies = 17 | { 18 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 19 | platforms = 20 | { 21 | linux = { 'ljsyscall >= 0.9' }, 22 | macosx = { 'ljsyscall >= 0.9' }, 23 | } 24 | } 25 | build = 26 | { 27 | type = "builtin", 28 | modules = 29 | { 30 | mmapfile = "src-lua/mmapfile.lua", 31 | ["mmapfile.unix"] = "src-lua/mmapfile/unix.lua", 32 | ["mmapfile.windows"] = "src-lua/mmapfile/windows.lua", 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /rockspecs/mmapfile-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "scm-1" 3 | source = 4 | { 5 | url = "git://github.com/geoffleyland/lua-mmapfile.git", 6 | branch = "master", 7 | } 8 | description = 9 | { 10 | summary = "Simple memory-mapped files", 11 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 12 | license = "MIT/X11", 13 | maintainer = "Geoff Leyland " 14 | } 15 | dependencies = 16 | { 17 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 18 | 'ljsyscall >= 0.9', 19 | } 20 | build = 21 | { 22 | type = "builtin", 23 | modules = 24 | { 25 | mmapfile = "lua/mmapfile.lua", 26 | }, 27 | } 28 | -------------------------------------------------------------------------------- /rockspecs/mmapfile-scm-2.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "scm-2" 3 | source = 4 | { 5 | url = "git://github.com/geoffleyland/lua-mmapfile.git", 6 | branch = "master", 7 | } 8 | description = 9 | { 10 | summary = "Simple memory-mapped files", 11 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 12 | license = "MIT/X11", 13 | maintainer = "Geoff Leyland " 14 | } 15 | dependencies = 16 | { 17 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 18 | platforms = 19 | { 20 | linux = { 'ljsyscall >= 0.9' }, 21 | macosx = { 'ljsyscall >= 0.9' }, 22 | } 23 | } 24 | build = 25 | { 26 | type = "builtin", 27 | modules = 28 | { 29 | mmapfile = "src-lua/mmapfile.lua", 30 | ["mmapfile.unix"] = "src-lua/mmapfile/unix.lua", 31 | ["mmapfile.windows"] = "src-lua/mmapfile/windows.lua", 32 | }, 33 | } 34 | -------------------------------------------------------------------------------- /rockspecs/mmapfile-scm-3.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "scm-3" 3 | source = 4 | { 5 | url = "git://github.com/geoffleyland/lua-mmapfile.git", 6 | branch = "master", 7 | } 8 | description = 9 | { 10 | summary = "Simple memory-mapped files", 11 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 12 | license = "MIT/X11", 13 | maintainer = "Geoff Leyland " 14 | } 15 | dependencies = 16 | { 17 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 18 | platforms = 19 | { 20 | linux = { 'ljsyscall >= 0.9' }, 21 | macosx = { 'ljsyscall >= 0.9' }, 22 | } 23 | } 24 | build = 25 | { 26 | type = "builtin", 27 | modules = 28 | { 29 | mmapfile = "src-lua/mmapfile.lua", 30 | ["mmapfile.unix"] = "src-lua/mmapfile/unix.lua", 31 | ["mmapfile.windows"] = "src-lua/mmapfile/windows.lua", 32 | ["mmapfile.string_list"] = "src-lua/mmapfile/string_list.lua", 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /rockspecs/mmapfile-scm-4.rockspec: -------------------------------------------------------------------------------- 1 | package = "mmapfile" 2 | version = "scm-4" 3 | source = 4 | { 5 | url = "git+https://github.com/geoffleyland/lua-mmapfile", 6 | branch = "master", 7 | } 8 | description = 9 | { 10 | summary = "Simple memory-mapped files", 11 | homepage = "http://github.com/geoffleyland/lua-mmapfile", 12 | license = "MIT/X11", 13 | maintainer = "Geoff Leyland " 14 | } 15 | dependencies = 16 | { 17 | 'lua == 5.1', -- should be "luajit >= 2.0.0" 18 | platforms = 19 | { 20 | linux = { 'ljsyscall >= 0.9' }, 21 | macosx = { 'ljsyscall >= 0.9' }, 22 | } 23 | } 24 | build = 25 | { 26 | type = "builtin", 27 | modules = 28 | { 29 | mmapfile = "src-lua/mmapfile.lua", 30 | ["mmapfile.unix"] = "src-lua/mmapfile/unix.lua", 31 | ["mmapfile.windows"] = "src-lua/mmapfile/windows.lua", 32 | ["mmapfile.string_list"] = "src-lua/mmapfile/string_list.lua", 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /spec/mmapfile_spec.lua: -------------------------------------------------------------------------------- 1 | -- luacheck: std max+busted 2 | 3 | local both_error_messages = 4 | { 5 | CREATE_BAD_DIRECTORY = 6 | { 7 | "mmapfile.create: Error creating 'there_is_no_directory_with_this_name/test':".. 8 | " No such file or directory", 9 | "mmapfile.create: Error creating 'there_is_no_directory_with_this_name/test':".. 10 | " The system cannot find the path specified.\r\n" 11 | }, 12 | OPEN_BAD_FILE = 13 | { 14 | "mmapfile.open: Error opening this_file_doesnt_exist: No such file or directory", 15 | "mmapfile.open: Error opening this_file_doesnt_exist: The system cannot find the file specified.\r\n" 16 | }, 17 | } 18 | 19 | local ei = jit.os == "Windows" and 2 or 1 20 | local errors = {} 21 | for k, v in pairs(both_error_messages) do 22 | errors[k] = v[ei] 23 | end 24 | 25 | 26 | describe("malloc", function() 27 | local mmapfile = require"mmapfile" 28 | 29 | test("malloc", function() 30 | local ptr = assert(mmapfile.malloc(1024, "long")) 31 | for i = 0, 1023 do 32 | ptr[i] = i + 1 33 | end 34 | for i = 0, 1023 do 35 | assert.equal(i+1, ptr[i]) 36 | end 37 | local ptr2 = assert(mmapfile.malloc(1024, "long", ptr)) 38 | for i = 0, 1023 do 39 | assert.equal(i+1, ptr2[i]) 40 | end 41 | 42 | mmapfile.free(ptr) 43 | mmapfile.free(ptr2) 44 | end) 45 | 46 | test("gcmalloc", function() 47 | do 48 | local ptr = assert(mmapfile.gcmalloc(1024, "long")) 49 | for i = 0, 1023 do 50 | ptr[i] = i + 2 51 | end 52 | for i = 0, 1023 do 53 | assert.equal(i+2, ptr[i]) 54 | end 55 | end 56 | collectgarbage() 57 | end) 58 | end) 59 | 60 | 61 | describe("mmap", function() 62 | local mmapfile = require"mmapfile" 63 | 64 | test("mmap", function() 65 | local ptr = assert(mmapfile.create("test", 1024, "uint32_t")) 66 | for i = 0, 1023 do 67 | ptr[i] = i + 3 68 | end 69 | mmapfile.close(ptr) 70 | local size 71 | ptr, size = mmapfile.open("test", "uint32_t") 72 | assert.equal(1024, size) 73 | for i = 0, 1023 do 74 | assert.equal(i+3, ptr[i]) 75 | end 76 | 77 | local ptr2 = assert(mmapfile.create("test2", 1024, "uint32_t", ptr)) 78 | mmapfile.close(ptr) 79 | 80 | for i = 0, 1023 do 81 | assert.equal(i+3, ptr2[i]) 82 | end 83 | mmapfile.close(ptr2) 84 | 85 | ptr2, size = mmapfile.open("test2", "uint32_t") 86 | assert.equal(1024, size) 87 | for i = 0, 1023 do 88 | assert.equal(i+3, ptr2[i]) 89 | end 90 | 91 | mmapfile.close(ptr2) 92 | end) 93 | 94 | test("can't create file", function() 95 | assert.has_error(function() mmapfile.create("there_is_no_directory_with_this_name/test", 1024, "uint32_t") end, 96 | errors.CREATE_BAD_DIRECTORY) 97 | end) 98 | 99 | test("can't open file", function() 100 | assert.has_error(function() mmapfile.open("this_file_doesnt_exist", "uint32_t") end, 101 | errors.OPEN_BAD_FILE) 102 | end) 103 | end) 104 | 105 | 106 | describe("mmap-rewrite", function() 107 | local mmapfile = require"mmapfile" 108 | 109 | test("mmap", function() 110 | local ptr = assert(mmapfile.create("test7", 1024, "uint32_t")) 111 | for i = 0, 1023 do 112 | ptr[i] = i + 7 113 | end 114 | mmapfile.close(ptr) 115 | local size 116 | ptr, size = mmapfile.open("test7", "uint32_t", "rw") 117 | assert.equal(1024, size) 118 | for i = 0, 1023 do 119 | assert.equal(i+7, ptr[i]) 120 | end 121 | for i = 0, 1023 do 122 | ptr[i] = i + 8 123 | end 124 | mmapfile.close(ptr) 125 | ptr, size = mmapfile.open("test7", "uint32_t", "rw") 126 | assert.equal(1024, size) 127 | for i = 0, 1023 do 128 | assert.equal(i+8, ptr[i]) 129 | end 130 | mmapfile.close(ptr) 131 | end) 132 | 133 | test("can't create file", function() 134 | assert.has_error(function() mmapfile.create("there_is_no_directory_with_this_name/test", 1024, "uint32_t") end, 135 | errors.CREATE_BAD_DIRECTORY) 136 | end) 137 | 138 | test("can't open file", function() 139 | assert.has_error(function() mmapfile.open("this_file_doesnt_exist", "uint32_t") end, 140 | errors.OPEN_BAD_FILE) 141 | end) 142 | end) 143 | 144 | 145 | 146 | describe("gcmmap", function() 147 | local mmapfile = require"mmapfile" 148 | 149 | test("mmap", function() 150 | do 151 | local ptr = assert(mmapfile.gccreate("test3", 1024, "uint32_t")) 152 | for i = 0, 1023 do 153 | ptr[i] = i + 4 154 | end 155 | end 156 | 157 | collectgarbage() 158 | 159 | do 160 | local ptr, size = mmapfile.gcopen("test3", "uint32_t") 161 | assert.equal(1024, size) 162 | for i = 0, 1023 do 163 | assert.equal(i+4, ptr[i]) 164 | end 165 | end 166 | collectgarbage() 167 | end) 168 | end) 169 | -------------------------------------------------------------------------------- /src-lua/config.ld: -------------------------------------------------------------------------------- 1 | project = "Lua-mmapfile" 2 | title = "Lua-mmapfile" 3 | description = "Lua-mmapfile provides a simple interface to mmap" 4 | format = "markdown" 5 | all = true 6 | -------------------------------------------------------------------------------- /src-lua/mmapfile.lua: -------------------------------------------------------------------------------- 1 | --- A simple interface to mmap or MapViewOfFile 2 | -- mmapfile uses `mmap` (on Unix) or `MapViewOfFileEx` (on Windows) to 3 | -- provide a way of quickly storing and loading data that's already in some 4 | -- kind of in-memory binary format. 5 | -- 6 | -- `create` creates a new file and maps new memory to that file. You can 7 | -- then write to the memory to write to the file. 8 | -- 9 | -- `open` opens an existing file and maps its contents to memory, returning 10 | -- a pointer to the memory and the length of the file. 11 | -- 12 | -- `close` syncs memory to the file, closes the file, and deletes the 13 | -- mapping between the memory and the file. 14 | -- 15 | -- The "gc" variants of `create` and `open` (`gccreate` and `gcopen`) set 16 | -- up a garbage collection callback for the pointer so that the file is 17 | -- correctly closed when the pointer is no longer referenced. Not 18 | -- appropriate if you might be storing the pointer in C, referencing it from 19 | -- unmanaged memory, or casting it to another type! 20 | -- 21 | -- All memory is mapped above 4G to try to keep away from the memory space 22 | -- LuaJIT uses. 23 | 24 | -- (c) Copyright 2014-2016 Geoff Leyland. 25 | -- See LICENSE for license information 26 | 27 | local ffi = require"ffi" 28 | local platform_mmapfile = (jit.os == "Windows" and require"mmapfile.windows") or 29 | require"mmapfile.unix" 30 | 31 | 32 | ------------------------------------------------------------------------------ 33 | 34 | --- Same as malloc, but set up a GC cleanup for the memory. 35 | -- @treturn pointer: the memory allocated 36 | local function gcmalloc( 37 | size, -- integer: number of bytes or `type`s to allocate. 38 | type, -- ?string: type to allocate 39 | data) -- ?pointer: data to copy to the mapped area. 40 | return ffi.gc(platform_mmapfile.malloc(size, type, data), platform_mmapfile.free) 41 | end 42 | 43 | 44 | ------------------------------------------------------------------------------ 45 | 46 | --- Same as create, but set up a GC cleanup for the memory and file. 47 | -- @treturn pointer: the memory allocated 48 | local function gccreate( 49 | filename, -- string: name of the file to create. 50 | size, -- integer: number of bytes or `type`s to allocate. 51 | type, -- ?string: type to allocate 52 | data) -- ?pointer: data to copy to the mapped area. 53 | return ffi.gc(platform_mmapfile.create(filename, size, type, data), platform_mmapfile.close) 54 | end 55 | 56 | 57 | --- Same as open, but set up a GC cleanup for the memory and file. 58 | -- @treturn pointer: the memory allocated. 59 | -- @treturn int: size of the file, in bytes or `type`s. 60 | local function gcopen( 61 | filename, -- string: name of the file to open. 62 | type, -- ?string: type to allocate 63 | mode, -- ?string: open mode for the file "r" or "rw" 64 | size, -- ?integer: size to map (in multiples of type). Default 65 | -- is file size 66 | offset) -- ?integer: offset into the file (default 0) 67 | local addr, actual_size = platform_mmapfile.open(filename, type, mode, size, offset) 68 | return ffi.gc(addr, platform_mmapfile.close), actual_size 69 | end 70 | 71 | 72 | ------------------------------------------------------------------------------ 73 | 74 | return 75 | { 76 | free = platform_mmapfile.free, 77 | malloc = platform_mmapfile.malloc, 78 | gcmalloc = gcmalloc, 79 | create = platform_mmapfile.create, 80 | gccreate = gccreate, 81 | open = platform_mmapfile.open, 82 | gcopen = gcopen, 83 | close = platform_mmapfile.close, 84 | } 85 | 86 | ------------------------------------------------------------------------------ 87 | 88 | -------------------------------------------------------------------------------- /src-lua/mmapfile/string_list.lua: -------------------------------------------------------------------------------- 1 | -- (c) Copyright 2014-2016 Geoff Leyland. 2 | -- See LICENSE for license information 3 | 4 | local ffi = require"ffi" 5 | local mmapfile = require"mmapfile" 6 | 7 | 8 | ------------------------------------------------------------------------------ 9 | 10 | local string_list = {} 11 | string_list.__index = string_list 12 | 13 | 14 | function string_list:new() 15 | return setmetatable({ count = 0, size = 0, strings = {}, map = {} }, self) 16 | end 17 | 18 | 19 | function string_list:add(s) 20 | if not self.map[s] then 21 | self.count = self.count + 1 22 | self.strings[self.count] = s 23 | self.map[s] = self.size 24 | self.size = self.size + #s + 1 25 | end 26 | end 27 | 28 | 29 | function string_list:write(filename) 30 | table.sort(self.strings) 31 | 32 | local string_ptr = mmapfile.gccreate(filename, self.size, "char") 33 | 34 | local offset = 0 35 | for _, s in ipairs(self.strings) do 36 | self.map[s] = offset 37 | ffi.copy(string_ptr + offset, s) 38 | offset = offset + #s + 1 39 | end 40 | 41 | return string_ptr 42 | end 43 | 44 | 45 | function string_list:offset(s) 46 | return self.map[s] 47 | end 48 | 49 | 50 | ------------------------------------------------------------------------------ 51 | 52 | local string_file = {} 53 | string_file.__index = string_file 54 | 55 | function string_list.read(filename) 56 | local o = { string_ptr = mmapfile.gcopen(filename, "char") } 57 | return setmetatable(o, string_file) 58 | end 59 | 60 | 61 | function string_file:get(index) 62 | return ffi.string(self.string_ptr + index) 63 | end 64 | 65 | 66 | ------------------------------------------------------------------------------ 67 | 68 | return string_list 69 | 70 | ------------------------------------------------------------------------------ 71 | -------------------------------------------------------------------------------- /src-lua/mmapfile/unix.lua: -------------------------------------------------------------------------------- 1 | --- A simple interface to mmap. 2 | -- mmapfile uses `mmap` to provide a way of quickly storing and loading data 3 | -- that's already in some kind of in-memory binary format. 4 | -- 5 | -- `create` creates a new file and maps new memory to that file. You can 6 | -- then write to the memory to write to the file. 7 | -- 8 | -- `open` opens an existing file and maps its contents to memory, returning 9 | -- a pointer to the memory and the length of the file. 10 | -- 11 | -- `close` syncs memory to the file, closes the file, and deletes the 12 | -- mapping between the memory and the file. 13 | -- 14 | -- The "gc" variants of `create` and `open` (`gccreate` and `gcopen`) set 15 | -- up a garbage collection callback for the pointer so that the file is 16 | -- correctly closed when the pointer is no longer referenced. Not 17 | -- appropriate if you might be storing the pointer in C, referencing it from 18 | -- unmanaged memory, or casting it to another type! 19 | -- 20 | -- All memory is mapped above 4G to try to keep away from the memory space 21 | -- LuaJIT uses. 22 | 23 | -- (c) Copyright 2014-2016 Geoff Leyland. 24 | -- See LICENSE for license information 25 | 26 | local S = require"syscall" 27 | local ffi = require"ffi" 28 | 29 | 30 | ------------------------------------------------------------------------------ 31 | 32 | local function assert(condition, message) 33 | if condition then return condition end 34 | message = message or "assertion failed" 35 | error(tostring(message), 2) 36 | end 37 | 38 | 39 | ------------------------------------------------------------------------------ 40 | 41 | --- Call mmap until we get an address higher that 4 gigabytes. 42 | -- mmapping over 4G means we don't step on LuaJIT's toes, and this usually 43 | -- works first time. 44 | -- See `man mmap` for explanation of parameters. 45 | -- @treturn pointer: the memory allocated. 46 | local function mmap_4G( 47 | size, -- integer: size to allocate in bytes 48 | prot, -- string: mmap's prot, as interpreted by syscall 49 | flags, -- string: mmap's flags, as interpreted by syscall 50 | fd, -- integer: file descriptor to map to 51 | offset) -- ?integer: offset into file to map 52 | offset = offset or 0 53 | local base = 4 * 1024 * 1024 * 1024 54 | local step = 2^math.floor(math.log(tonumber(size)) / math.log(2)) 55 | local addr 56 | for _ = 1, 1024 do 57 | addr = S.mmap(ffi.cast("void*", base), size, prot, flags, fd, offset) 58 | if addr >= ffi.cast("void*", 4 * 1024 * 1024 * 1024) then break end 59 | S.munmap(addr, size) 60 | base = base + step 61 | end 62 | return addr 63 | end 64 | 65 | 66 | ------------------------------------------------------------------------------ 67 | 68 | local malloced_sizes = {} 69 | 70 | 71 | --- "Map" some anonymous memory that's not mapped to a file. 72 | -- This makes mmap behave a bit like malloc, except that we can persuade it 73 | -- to give us memory above 4G, and malloc will be a very, very slow allocator 74 | -- so only use it infrequently on big blocks of memory. 75 | -- This is really just a hack to get us memory above 4G. There's probably a 76 | -- better solution. 77 | -- @treturn pointer: the memory allocated. 78 | local function malloc( 79 | size, -- integer: number of bytes or `type`s to allocate. 80 | type, -- ?string: type to allocate 81 | data) -- ?pointer: data to copy to the mapped area. 82 | 83 | if type then 84 | size = size * ffi.sizeof(type) 85 | end 86 | 87 | local addr = assert(mmap_4G(size, "read, write", "anon, shared")) 88 | 89 | malloced_sizes[tostring(ffi.cast("void*", addr))] = size 90 | 91 | if data then 92 | ffi.copy(addr, data, size) 93 | end 94 | 95 | if type then 96 | return ffi.cast(type.."*", addr) 97 | else 98 | return addr 99 | end 100 | end 101 | 102 | 103 | --- Free memory mapped with mmapfile.malloc 104 | -- Just munmaps the memory. 105 | local function free( 106 | addr) -- pointer: the mapped address to unmap. 107 | local s = tostring(ffi.cast("void*", addr)) 108 | local size = assert(malloced_sizes[s], "no mmapped block at this address") 109 | assert(S.munmap(addr, size)) 110 | end 111 | 112 | ------------------------------------------------------------------------------ 113 | 114 | local open_fds = {} 115 | 116 | 117 | --- Close a mapping between a file and an address. 118 | -- `msync` the memory to its associated file, `munmap` the memory, and close 119 | -- the file. 120 | local function close( 121 | addr) -- pointer: the mapped address to unmap. 122 | local s = tostring(ffi.cast("void*", addr)) 123 | local fd = assert(open_fds[s], "no file open for this address") 124 | open_fds[s] = nil 125 | 126 | -- it seems that file descriptors get closed before final __gc calls in 127 | -- some exit scenarios, so we don't worry too much if we can't 128 | -- stat the fd 129 | local st = fd:stat() 130 | if st then 131 | assert(S.msync(addr, st.size, "sync")) 132 | assert(S.munmap(addr, st.size)) 133 | assert(fd:close()) 134 | end 135 | end 136 | 137 | 138 | --- Allocate memory and create a new file mapped to it. 139 | -- Use create to set aside an area of memory to write to a file. 140 | -- If `type` is supplied then the pointer to the allocated memory is cast 141 | -- to the correct type, and `size` is the number of `type`, not bytes, 142 | -- to allocate. 143 | -- If `data` is supplied, then the data at `data` is copied into the mapped 144 | -- memory (and so written to the file). It might make more sense just to 145 | -- map the pointer `data` directly to the file, but that might require `data` 146 | -- to be on a page boundary. 147 | -- The file descriptor is saved in a table keyed to the address allocated 148 | -- so that close can find the write fd to close when the memory is unmapped. 149 | -- @treturn pointer: the memory allocated. 150 | local function create( 151 | filename, -- string: name of the file to create. 152 | size, -- integer: number of bytes or `type`s to allocate. 153 | type, -- ?string: type to allocate 154 | data) -- ?pointer: data to copy to the mapped area. 155 | local fd, message = S.open(filename, "RDWR, CREAT", "RUSR, WUSR, RGRP, ROTH") 156 | 157 | if not fd then 158 | error(("mmapfile.create: Error creating '%s': %s"):format(filename, message)) 159 | end 160 | 161 | if type then 162 | size = size * ffi.sizeof(type) 163 | end 164 | 165 | -- lseek gets stroppy if we try to seek the -1th byte, so let's just say 166 | -- all files are at least one byte, even if theres's no actual data. 167 | size = math.max(size, 1) 168 | assert(fd:lseek(size-1, "set")) 169 | assert(fd:write(ffi.new("char[1]", 0), 1)) 170 | 171 | local addr = assert(mmap_4G(size, "read, write", "file, shared", fd)) 172 | 173 | open_fds[tostring(ffi.cast("void*", addr))] = fd 174 | 175 | if data then 176 | ffi.copy(addr, data, size) 177 | end 178 | 179 | if type then 180 | return ffi.cast(type.."*", addr) 181 | else 182 | return addr 183 | end 184 | end 185 | 186 | 187 | --- Map an existing file to an area of memory. 188 | -- If `type` is present, the the pointer returned is cast to the `type*` and 189 | -- the size returned is the number of `types`, not bytes. 190 | -- @treturn pointer: the memory allocated. 191 | -- @treturn int: size of the file, in bytes or `type`s. 192 | local function open( 193 | filename, -- string: name of the file to open. 194 | type, -- ?string: type to allocate (default void*) 195 | mode, -- ?string: open mode for the file "r" or "rw" (default "r") 196 | size, -- ?integer: size to map (in multiples of type). Default 197 | -- is file size 198 | offset) -- ?integer: offset into the file (default 0) 199 | offset = offset or 0 200 | 201 | mode = mode or "r" 202 | local filemode, mapmode 203 | if mode == "r" then 204 | filemode = "rdonly" 205 | mapmode = "read" 206 | elseif mode == "rw" then 207 | filemode = "rdwr" 208 | mapmode = "read, write" 209 | else 210 | return nil, "unknown read/write mode" 211 | end 212 | 213 | local fd, message = S.open(filename, filemode, 0) 214 | if not fd then 215 | error(("mmapfile.open: Error opening %s: %s"):format(filename, message)) 216 | end 217 | 218 | if not size then 219 | local st = assert(fd:stat()) 220 | size = st.size 221 | elseif type then 222 | size = size * ffi.sizeof(type) 223 | end 224 | 225 | local addr = assert(mmap_4G(size, mapmode, "file, shared", fd, offset)) 226 | 227 | open_fds[tostring(ffi.cast("void*", addr))] = fd 228 | 229 | if type then 230 | return ffi.cast(type.."*", addr), math.floor(size / ffi.sizeof(type)) 231 | else 232 | return addr, size 233 | end 234 | end 235 | 236 | 237 | ------------------------------------------------------------------------------ 238 | 239 | return 240 | { 241 | free = free, 242 | malloc = malloc, 243 | create = create, 244 | open = open, 245 | close = close, 246 | } 247 | 248 | ------------------------------------------------------------------------------ 249 | 250 | -------------------------------------------------------------------------------- /src-lua/mmapfile/windows.lua: -------------------------------------------------------------------------------- 1 | --- A simple interface to MapViewOfFile. 2 | -- mmapfile uses `MapViewOfFileEx` to provide a way of quickly storing and loading data 3 | -- that's already in some kind of in-memory binary format. 4 | -- 5 | -- `create` creates a new file and maps new memory to that file. You can 6 | -- then write to the memory to write to the file. 7 | -- 8 | -- `open` opens an existing file and maps its contents to memory, returning 9 | -- a pointer to the memory and the length of the file. 10 | -- 11 | -- `close` syncs memory to the file, closes the file, and deletes the 12 | -- mapping between the memory and the file. 13 | -- 14 | -- All memory is mapped above 4G to try to keep away from the memory space 15 | -- LuaJIT uses. 16 | 17 | -- (c) Copyright 2014-2016 Geoff Leyland. 18 | -- See LICENSE for license information 19 | 20 | local ffi = require"ffi" 21 | 22 | ------------------------------------------------------------------------------ 23 | 24 | ffi.cdef[[ 25 | typedef uint32_t DWORD; 26 | typedef const char *LPCTSTR; 27 | typedef void *HANDLE; 28 | typedef int64_t LARGE_INTEGER; 29 | typedef int BOOL; 30 | 31 | DWORD GetLastError(); 32 | DWORD FormatMessageA( 33 | DWORD dwFlags, 34 | const void* lpSource, 35 | DWORD dwMessageId, 36 | DWORD dwLanguageId, 37 | char* lpBuffer, 38 | DWORD nSize, 39 | va_list *Arguments); 40 | HANDLE CreateFileA( 41 | LPCTSTR lpFileName, 42 | DWORD dwDesiredAccess, 43 | DWORD dwShareMode, 44 | void* lpSecurityAttributes, 45 | DWORD dwCreationDisposition, 46 | DWORD dwFlagsAndAttributes, 47 | HANDLE hTemplateFile); 48 | HANDLE CreateFileMappingA( 49 | HANDLE hFile, 50 | void* lpAttributes, 51 | DWORD flProtect, 52 | DWORD dwMaximumSizeHigh, 53 | DWORD dwMaximumSizeLow, 54 | LPCTSTR lpName); 55 | void* MapViewOfFileEx( 56 | HANDLE hFileMappingObject, 57 | DWORD dwDesiredAccess, 58 | DWORD dwFileOffsetHigh, 59 | DWORD dwFileOffsetLow, 60 | size_t dwNumberOfBytesToMap, 61 | void *lpBaseAddress); 62 | long UnmapViewOfFile(void* lpBaseAddress); 63 | long CloseHandle(HANDLE hObject); 64 | BOOL GetFileSizeEx( 65 | HANDLE hFile, 66 | LARGE_INTEGER* lpFileSize); 67 | 68 | static const DWORD FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000; 69 | static const DWORD FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200; 70 | 71 | static const DWORD GENERIC_READ = 0x80000000; 72 | static const DWORD GENERIC_WRITE = 0x40000000; 73 | 74 | static const DWORD OPEN_EXISTING = 0x00000003; 75 | static const DWORD CREATE_ALWAYS = 0x00000002; 76 | 77 | static const DWORD FILE_ATTRIBUTE_ARCHIVE = 0x00000020; 78 | static const DWORD FILE_FLAG_RANDOM_ACCESS = 0x10000000; 79 | 80 | static const DWORD FILE_MAP_ALL_ACCESS = 0x000f001f; 81 | static const DWORD FILE_MAP_READ = 0x00000004; 82 | 83 | static const DWORD PAGE_READWRITE = 0x00000004; 84 | static const DWORD PAGE_READONLY = 0x00000002; 85 | 86 | static const DWORD FILE_SHARE_READ = 0x00000001; 87 | ]] 88 | 89 | local INVALID_HANDLE_VALUE = ffi.cast("HANDLE", -1) -- I can't work out how to define this in the cdefs. 90 | 91 | 92 | ------------------------------------------------------------------------------ 93 | 94 | local error_buffer = ffi.new("char[1024]") 95 | local function last_error_string() 96 | local code = ffi.C.GetLastError() 97 | local length = ffi.C.FormatMessageA( 98 | bit.bor(ffi.C.FORMAT_MESSAGE_FROM_SYSTEM, ffi.C.FORMAT_MESSAGE_IGNORE_INSERTS), 99 | nil, code, 0, error_buffer, 1023, nil) 100 | return ffi.string(error_buffer, length) 101 | end 102 | 103 | 104 | local size_buffer = ffi.new("LARGE_INTEGER[1]") 105 | local function get_file_size(file) 106 | local ok = ffi.C.GetFileSizeEx(file, size_buffer) 107 | if ok == 0 then 108 | error(("mmapfile.get_file_size: error getting file size: %s"):format(last_error_string())) 109 | end 110 | return tonumber(size_buffer[0]) 111 | end 112 | 113 | 114 | ------------------------------------------------------------------------------ 115 | 116 | --- Call mmap until we get an address higher that 4 gigabytes. 117 | -- mmapping over 4G means we don't step on LuaJIT's toes, and this usually 118 | -- works first time. 119 | -- See `man mmap` for explanation of parameters. 120 | -- @treturn pointer: the memory allocated. 121 | local function mmap_4G( 122 | map, -- file map to map to 123 | size, -- integer: size to allocate in bytes 124 | access, -- string: mmap's prot, as interpreted by syscall 125 | offset) -- ?integer: offset into file to map 126 | offset = offset or 0 127 | local base = 4 * 1024 * 1024 * 1024 128 | local step = 0x10000 129 | local addr 130 | for _ = 1, 32 do 131 | addr = ffi.C.MapViewOfFileEx(map, access, 0, offset, size, ffi.cast("void*", base)) 132 | 133 | if addr >= ffi.cast("void*", 4 * 1024 * 1024 * 1024) then break end 134 | if addr ~= nil then 135 | ffi.C.UnmapViewOfFile(addr) 136 | end 137 | base = base + step 138 | step = math.min(2^26, step * 2) -- 64MB should be a pretty good step, right? 139 | end 140 | return addr 141 | end 142 | 143 | 144 | ------------------------------------------------------------------------------ 145 | 146 | local malloced_maps = {} 147 | 148 | --- "Map" some anonymous memory that's not mapped to a file. 149 | -- This makes mmap behave a bit like malloc, except that we can persuade it 150 | -- to give us memory above 4G, and malloc will be a very, very slow allocator 151 | -- so only use it infrequently on big blocks of memory. 152 | -- This is really just a hack to get us memory above 4G. There's probably a 153 | -- better solution. 154 | -- @treturn pointer: the memory allocated. 155 | local function malloc( 156 | size, -- integer: number of bytes or `type`s to allocate. 157 | type, -- ?string: type to allocate 158 | data) -- ?pointer: data to copy to the mapped area. 159 | 160 | if type then 161 | size = size * ffi.sizeof(type) 162 | end 163 | 164 | local map = ffi.C.CreateFileMappingA(INVALID_HANDLE_VALUE, nil, ffi.C.PAGE_READWRITE, 0, size, nil) 165 | if map == nil then 166 | error(("mmapfile.malloc: Error: %s"):format(last_error_string())) 167 | end 168 | 169 | local addr = mmap_4G(map, size, ffi.C.FILE_MAP_ALL_ACCESS, 0) 170 | if addr == nil then 171 | error(("mmapfile.malloc: Error: %s"):format(last_error_string())) 172 | end 173 | 174 | malloced_maps[tostring(ffi.cast("void*", addr))] = map 175 | 176 | if data then 177 | ffi.copy(addr, data, size) 178 | end 179 | 180 | if type then 181 | return ffi.cast(type.."*", addr) 182 | else 183 | return addr 184 | end 185 | end 186 | 187 | 188 | --- Free memory mapped with mmapfile.malloc 189 | -- Just munmaps the memory. 190 | local function free( 191 | addr) -- pointer: the mapped address to unmap. 192 | local s = tostring(ffi.cast("void*", addr)) 193 | local map = assert(malloced_maps[s], "no mmapped block at this address") 194 | ffi.C.UnmapViewOfFile(addr) 195 | ffi.C.CloseHandle(map) 196 | end 197 | 198 | 199 | ------------------------------------------------------------------------------ 200 | 201 | local open_fds = {} 202 | 203 | 204 | --- Close a mapping between a file and an address. 205 | -- `msync` the memory to its associated file, `munmap` the memory, and close 206 | -- the file. 207 | local function close( 208 | addr) -- pointer: the mapped address to unmap. 209 | local s = tostring(ffi.cast("void*", addr)) 210 | local fd = assert(open_fds[s], "no file open for this address") 211 | open_fds[s] = nil 212 | 213 | ffi.C.UnmapViewOfFile(addr) 214 | ffi.C.CloseHandle(fd.map) 215 | ffi.C.CloseHandle(fd.fd) 216 | end 217 | 218 | 219 | --- Allocate memory and create a new file mapped to it. 220 | -- Use create to set aside an area of memory to write to a file. 221 | -- If `type` is supplied then the pointer to the allocated memory is cast 222 | -- to the correct type, and `size` is the number of `type`, not bytes, 223 | -- to allocate. 224 | -- If `data` is supplied, then the data at `data` is copied into the mapped 225 | -- memory (and so written to the file). It might make more sense just to 226 | -- map the pointer `data` directly to the file, but that might require `data` 227 | -- to be on a page boundary. 228 | -- The file descriptor is saved in a table keyed to the address allocated 229 | -- so that close can find the write fd to close when the memory is unmapped. 230 | -- @treturn pointer: the memory allocated. 231 | local function create( 232 | filename, -- string: name of the file to create. 233 | size, -- integer: number of bytes or `type`s to allocate. 234 | type, -- ?string: type to allocate 235 | data) -- ?pointer: data to copy to the mapped area. 236 | local fd = ffi.C.CreateFileA(filename, 237 | bit.bor(ffi.C.GENERIC_READ, ffi.C.GENERIC_WRITE), 0, nil, ffi.C.CREATE_ALWAYS, 238 | bit.bor(ffi.C.FILE_ATTRIBUTE_ARCHIVE, ffi.C.FILE_FLAG_RANDOM_ACCESS), nil) 239 | 240 | if fd == INVALID_HANDLE_VALUE then 241 | error(("mmapfile.create: Error creating '%s': %s"):format(filename, last_error_string())) 242 | end 243 | 244 | if type then 245 | size = size * ffi.sizeof(type) 246 | end 247 | 248 | local map = ffi.C.CreateFileMappingA(fd, nil, ffi.C.PAGE_READWRITE, 0, size, nil) 249 | if map == nil then 250 | error(("mmapfile.create: Error creating %s: %s"):format(filename, last_error_string())) 251 | end 252 | 253 | local addr = mmap_4G(map, size, ffi.C.FILE_MAP_ALL_ACCESS, 0) 254 | if addr == nil then 255 | error(("mmapfile.create: Error creating %s: %s"):format(filename, last_error_string())) 256 | end 257 | 258 | open_fds[tostring(ffi.cast("void*", addr))] = { fd = fd, map = map } 259 | 260 | if data then 261 | ffi.copy(addr, data, size) 262 | end 263 | 264 | if type then 265 | return ffi.cast(type.."*", addr) 266 | else 267 | return addr 268 | end 269 | end 270 | 271 | 272 | --- Map an existing file to an area of memory. 273 | -- If `type` is present, the the pointer returned is cast to the `type*` and 274 | -- the size returned is the number of `types`, not bytes. 275 | -- @treturn pointer: the memory allocated. 276 | -- @treturn int: size of the file, in bytes or `type`s. 277 | local function open( 278 | filename, -- string: name of the file to open. 279 | type, -- ?string: type to allocate (default void*) 280 | mode, -- ?string: open mode for the file "r" or "rw" (default "r") 281 | size, -- ?integer: size to map (in multiples of type). Default 282 | -- is file size 283 | offset) -- ?integer: offset into the file (default 0) 284 | offset = offset or 0 285 | 286 | mode = mode or "r" 287 | local filemode, sharemode, mapmode, ptrmode 288 | if mode == "r" then 289 | filemode = ffi.C.GENERIC_READ 290 | sharemode = ffi.C.FILE_SHARE_READ 291 | mapmode = ffi.C.PAGE_READONLY 292 | ptrmode = ffi.C.FILE_MAP_READ 293 | elseif mode == "rw" then 294 | filemode = bit.bor(ffi.C.GENERIC_READ, ffi.C.GENERIC_WRITE) 295 | sharemode = 0 296 | mapmode = ffi.C.PAGE_READWRITE 297 | ptrmode = ffi.C.FILE_MAP_ALL_ACCESS 298 | else 299 | return nil, "unknown read/write mode" 300 | end 301 | 302 | local fd = ffi.C.CreateFileA(filename, 303 | filemode, sharemode, nil, ffi.C.OPEN_EXISTING, 304 | bit.bor(ffi.C.FILE_ATTRIBUTE_ARCHIVE, ffi.C.FILE_FLAG_RANDOM_ACCESS), nil) 305 | if fd == INVALID_HANDLE_VALUE then 306 | error(("mmapfile.open: Error opening %s: %s"):format(filename, last_error_string())) 307 | end 308 | 309 | if not size then 310 | size = get_file_size(fd) 311 | elseif type then 312 | size = size * ffi.sizeof(type) 313 | end 314 | 315 | local map = ffi.C.CreateFileMappingA(fd, nil, mapmode, 0, size, nil) 316 | if map == nil then 317 | error(("mmapfile.open: Error opening %s: %s"):format(filename, last_error_string())) 318 | end 319 | 320 | local addr = mmap_4G(map, 0, ptrmode, offset) 321 | if addr == nil then 322 | error(("mmapfile.open: Error opening %s: %s"):format(filename, last_error_string())) 323 | end 324 | 325 | open_fds[tostring(ffi.cast("void*", addr))] = { fd = fd, map = map } 326 | 327 | if type then 328 | return ffi.cast(type.."*", addr), math.floor(size / ffi.sizeof(type)) 329 | else 330 | return addr, size 331 | end 332 | end 333 | 334 | 335 | ------------------------------------------------------------------------------ 336 | 337 | return 338 | { 339 | free = free, 340 | malloc = malloc, 341 | create = create, 342 | open = open, 343 | close = close, 344 | } 345 | 346 | ------------------------------------------------------------------------------ 347 | 348 | --------------------------------------------------------------------------------