├── .gitignore ├── Makefile ├── README.md ├── TODO ├── mmap_access_mode.h ├── mmap_allocator.h ├── mmap_exception.h ├── mmap_file_pool.cpp ├── mmap_file_pool.h ├── mmappable_vector.h ├── test_allocator.cpp └── test_mmap_fixed.c /.gitignore: -------------------------------------------------------------------------------- 1 | testfile 2 | testfile2 3 | test_allocator 4 | *.o 5 | .*.swp 6 | *.so 7 | *.a 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CPPFLAGS=-g -Wall -fPIC 2 | CFLAGS=-g -Wall -fPIC 3 | 4 | # Enable to test with GCC 3.4 5 | # CXX=g++34 6 | 7 | PREFIX=/usr 8 | 9 | SOURCES=mmap_file_pool.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h mmappable_vector.h mmap_exception.h 10 | 11 | HEADERS=mmap_access_mode.h mmap_allocator.h mmap_file_pool.h mmap_exception.h 12 | LIBRARIES=libmmap_allocator.so libmmap_allocator.a 13 | 14 | SRC_INSTALL_TARGET_DIR=/home/johannes/re3 15 | 16 | all: test_allocator mmap_file_pool.o $(LIBRARIES) 17 | 18 | debug: CPPFLAGS+=-DMMAP_ALLOCATOR_DEBUG=1 19 | debug: CFLAGS+=-DMMAP_ALLOCATOR_DEBUG=1 20 | debug: clean all 21 | 22 | libmmap_allocator.so: mmap_file_pool.o 23 | g++ -shared -o libmmap_allocator.so mmap_file_pool.o 24 | 25 | libmmap_allocator.a: mmap_file_pool.o 26 | ar r libmmap_allocator.a mmap_file_pool.o 27 | 28 | install_sources: $(SOURCES) 29 | cp $(SOURCES) $(SRC_INSTALL_TARGET_DIR) 30 | 31 | install: all 32 | install -m 644 $(HEADERS) $(PREFIX)/include 33 | install -m 755 $(LIBRARIES) $(PREFIX)/lib 34 | 35 | test: all 36 | @echo "Running mmap allocator regression test suite." 37 | bash -c 'export LD_LIBRARY_PATH=. ; ./test_allocator' 38 | 39 | debugtest: debug 40 | @echo "Running mmap allocator regression test suite with verbose enabled." 41 | bash -c 'export LD_LIBRARY_PATH=. ; ./test_allocator' 42 | 43 | test_allocator: mmap_allocator.h mmap_file_pool.o test_allocator.o $(LIBRARIES) 44 | g++ test_allocator.o -L. -lmmap_allocator -o test_allocator 45 | 46 | test_mmap_fixed: test_mmap_fixed.c 47 | gcc $(CFLAGS) test_mmap_fixed.c -o test_mmap_fixed 48 | 49 | clean: 50 | rm -f test_allocator test_mmap_fixed testfile testfile2 *.o $(LIBRARIES) 51 | 52 | mmap_file_pool.o: mmap_file_pool.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h mmappable_vector.h mmap_exception.h 53 | 54 | test_allocator.o: test_allocator.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h mmappable_vector.h mmap_exception.h 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | mmap_allocator - A STL allocator that mmaps files 2 | ------------------------------------------------- 3 | 4 | Introduction 5 | ------------ 6 | 7 | When reading large files (>100MB) into memory, read() calls are usually 8 | not very space and time efficient, since the whole data is copiied at 9 | least once. Furthermore, when using STL containers (like a vector for 10 | example), data is copiied another time unless you specify the location 11 | of the vector content as parameter to read(). 12 | 13 | It would be nice to tell the vector a filename and have the vector mmap 14 | the file directly. This not only avoids the read() copiing (and the 15 | STL vector copiing) but also allows different processes that read the 16 | same file to see the same physical memory. Fortunately STL foresees 17 | an interface to do exactly this. 18 | 19 | Put short, if you need to handle big files that contain unstructured data 20 | (like doubles or even text files), mmap_allocator is worth a try. It was 21 | written as part of a consulting project I did for a large Austrian bank. 22 | 23 | License 24 | ------- 25 | 26 | This code is (c) Johannes Thoma 2012 and is LGPL. Please write me an email to 27 | johannes.thoma@gmx.at if you find this useful, or if you find bugs or want 28 | to extend the library. 29 | 30 | How it works 31 | ------------ 32 | 33 | Many STL container classes accept allocators as part of their instantiation 34 | and construction. The default allocator simply does a malloc on allocate and 35 | a free on deallocate. By subclassing the default allocator and replacing the 36 | allocate and deallocate methods with code that memory maps a file and return 37 | a pointer to the mmaped area, we can create STL containers that use memory 38 | mapped regions directly. 39 | 40 | Usage 41 | ----- 42 | 43 | To build this, do a 44 | 45 | make 46 | 47 | followed by a 48 | 49 | make install 50 | 51 | When compiling, include the mmappable_vector.h file and link with 52 | the -lmmap_allocator library. 53 | 54 | Example 55 | ------- 56 | 57 | Suppose you have a file with 1024 ints. Then: 58 | 59 | mmappable_vector my_vector; 60 | 61 | declares a vector that uses mmap_allocator to mmap the file. By calling: 62 | 63 | my_vector.mmap_file("testfile", READ_ONLY, 0, 1024); 64 | 65 | the mmappable_vector class will call allocate of the mmap_allocator 66 | class, which in turn will mmap the file and set the content of the 67 | vector to the mmapped area. The file's content can then be accessed 68 | the usual way (my_vector[i]). 69 | 70 | Use 71 | 72 | my_vector.munmap_file(); 73 | 74 | to drop the mapping (it may remain in an internal cache, however). After 75 | this call all accesses of mmappable_vector elements are invalid, same 76 | goes for the iterators. 77 | 78 | Do not forget to: 79 | 80 | using namespace mmap_allocator_namespace; 81 | 82 | and include: 83 | 84 | #include "mmappable_vector.h" 85 | 86 | but you probably know that yourself ;) 87 | 88 | Please see the test_allocator.cpp file for more examples 89 | (I used Testdriven development for this project and can highly recommend 90 | this development model). 91 | 92 | Mode 93 | ---- 94 | 95 | As part of the construction of the mmap_allocator object a mode field can be 96 | specified. It does the following: 97 | 98 | * DEFAULT_STL_ALLOCATOR: Default STL allocator (malloc based). Reason 99 | is to have containers that do both and are compatible 100 | * READ_ONLY: Readonly modus. Segfaults when vector content is written to. 101 | * READ_WRITE_PRIVATE: Read/write access, writes are not propagated to disk. 102 | * READ_WRITE_SHARED: Read/write access, writes are propagated to disk 103 | (file is modified) 104 | 105 | The offset parameter must be page aligned (PAGE_SIZE, usually 4K or 8K), 106 | else mmap will return an error. 107 | 108 | Mmap file pool 109 | -------------- 110 | 111 | Beginning with 0.3, mmap allocator uses a pool of mmaped files. If you 112 | map the same file with the same access mode again it gets the same 113 | (virtual and physical) memory assigned as it was mapped previously. 114 | We needed this because our program (that uses mmap_allocator) mmaps 115 | a file in many small chunks (10K junks) which eventually lead to a 116 | Out of filedescriptors error (when mapping a 100Meg file in 10K junks). 117 | 118 | Following flags can be passed (by |'ing them together) to the mmap_file 119 | method: 120 | 121 | * MAP_WHOLE_FILE: Set this flag when you know that you need 122 | the whole (or most parts of the) file later and only want to 123 | request a part of it now. 124 | 125 | * ALLOW_REMAP: Set this flag if you are allocating a vector 126 | from a mmapped file and you know that you do not need previous 127 | mappings any more. Normally this is used when the file size 128 | changes (in particular when it grows). Be aware, however that 129 | this could invalidate all allocations for that file that have 130 | been made before. 131 | 132 | * BYPASS_FILE_POOL: Set this flag if you want a per-vector 133 | private mapping. This is useful in conjunction with READ_WRITE_PRIVATE 134 | mappings. 135 | 136 | Mmappable vector class 137 | ---------------------- 138 | 139 | Beginning with version 0.4, there is a special std::vector subclass 140 | for use with mmapped vectors. This was necessary because there is 141 | no standard way to set the size of an STL vector without initializing 142 | the content (which is not wanted since the content is coming from the 143 | mmapped file). From now on, please use the mmapped_vector class for 144 | using mmap_allocator. 145 | 146 | The old method by using the mmap_allocators constructor to set the 147 | parameters for file mapping is still supported, however deprecated. 148 | 149 | READ_WRITE caveats 150 | ------------------ 151 | 152 | Unlike reading from a file, a mmappable_vector that is mapped via 153 | the mmap file pool contains all changes already made to the content 154 | before. Suppose if you have: 155 | 156 | mmappable_vector p; 157 | if (method == MMAP) { 158 | p.mmap_file("testfile", READ_WRITE_PRIVATE, 0, filesize("testfile")); 159 | } else { 160 | readFromFileIntoVector(p, "testfile", READ_WRITE_PRIVATE, 0, filesize("testfile") 161 | } 162 | 163 | and then do something like: 164 | 165 | for (it = p.begin();it != p.end();it++) { 166 | *it += 1; 167 | } 168 | 169 | This will do not what you would expect at first glance when being called 170 | twice. When the vector maps the file for the second time, it will map it 171 | from the file pool and hence already have the values increased by one. 172 | 173 | To avoid this, use the bypass_file_pool flag which will cause the file 174 | being mmapped another time with different virtual memory mappings. 175 | 176 | Exceptions 177 | ---------- 178 | 179 | There is a custom exception class mmap_allocator_exception which is 180 | used at various places, for example when the file to be mapped is 181 | not found. Use the e.message() method to find out what happened. 182 | 183 | Conversion function templates 184 | ----------------------------- 185 | 186 | To convert between a STL vector and a mmappable vector, use the 187 | to_std_vector(mappable_vector) and to_mmappable_vector(std_vector) 188 | function templates. However, try to avoid this because this will 189 | copy the whole vector contents. 190 | 191 | Verbosity 192 | --------- 193 | 194 | To debug the allocator, there are two ways: 195 | 196 | * call set_verbosity() with a positive value at runtime. 197 | * define MMAP_ALLOCATOR_DEBUG as a positive value at compile time. 198 | 199 | The latter is done by make test. 200 | 201 | Version history 202 | --------------- 203 | 204 | * 0.1.0, first release. 205 | * 0.2.0, some interface changes. 206 | * 0.3.0, mmaped file pool. 207 | * 0.3.1, do not remap files when area fits in already mapped file. 208 | * 0.3.2, never use MAP_FIXED. 209 | * 0.3.3, bugfix in computing pointers. 210 | * 0.4.0, mmapped_vector class. 211 | * 0.5.0, cleaner interface, illegal offset bugfix. 212 | * 0.5.1, bypass_file_pool flag. 213 | * 0.5.2, changed bool parameters to flags argument. 214 | * 0.5.3, set_verbosity() to toggle debug output. 215 | * 0.6.0, to_stl_vector function template. 216 | * 0.6.1, flags bugfix. 217 | * 0.7.0, moved mmappable_vector to separate header. 218 | * 0.8.0, to_mmappable_vector conversion function. 219 | * 0.8.1, exception now knows what() method. 220 | * 0.9.0, more standard conformant exception handling. 221 | * 0.9.1, fixed a permission bug in MMAP_READWRITE_PRIVATE. 222 | * 0.10.0, KEEP_FOREVER flag: never close any file. 223 | * 0.10.1, fixed bug with allocating/deallocating 0 bytes. 224 | 225 | Author 226 | ------ 227 | 228 | This was written by Johannes Thoma 229 | Thanks to Piotr Nycz from stackoverflow for contributing 230 | the constructor function template in mmappable_vector.h 231 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | TODO for mmap_allocator 2 | ----------------------- 3 | 4 | Done: 5 | 6 | Option to not use mempool (bypass_file_pool). 7 | 8 | Convert bool parameters to bit flag. 9 | 10 | Have a option to set verbosity at runtime, instead of having to recompile. 11 | 12 | Compile as a shared library. 13 | 14 | make install should do what one expects (-> install_sources). 15 | 16 | Have more converters between mmappable_vectors and plain STL vectors. 17 | 18 | For 0.7: 19 | 20 | Support for STL strings. 21 | 22 | Move mmappable_vector to separate file. 23 | 24 | Later: 25 | 26 | Support for other platforms 27 | At least test different gcc / libstdc++ implementation. 28 | 29 | Allow remapping of already mapped vectors. 30 | 31 | Conversion function template from std vectors to mmapped vectors (if possible): 32 | Only can convert from mmappable_vectors to std::vectors, don't 33 | know how to convert the other way. 34 | 35 | std:string support 36 | -------------------------------------------------------------------------------- /mmap_access_mode.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMAP_ACCESS_MODE 2 | #define _MMAP_ACCESS_MODE 3 | 4 | #include 5 | 6 | #define ALIGN_TO_PAGE(x) ((x) & ~(getpagesize() - 1)) 7 | #define UPPER_ALIGN_TO_PAGE(x) ALIGN_TO_PAGE((x)+(getpagesize()-1)) 8 | #define OFFSET_INTO_PAGE(x) ((x) & (getpagesize() - 1)) 9 | 10 | namespace mmap_allocator_namespace 11 | { 12 | enum access_mode { 13 | DEFAULT_STL_ALLOCATOR, /* Default STL allocator (malloc based). Reason is to have containers that do both and are compatible */ 14 | READ_ONLY, /* Readonly modus. Segfaults when vector content is written to */ 15 | READ_WRITE_PRIVATE, /* Read/write access, writes are not propagated to disk */ 16 | READ_WRITE_SHARED /* Read/write access, writes are propagated to disk (file is modified) */ 17 | }; 18 | 19 | enum allocator_flags { 20 | MAP_WHOLE_FILE = 1, 21 | ALLOW_REMAP = 2, 22 | BYPASS_FILE_POOL = 4, 23 | KEEP_FOREVER = 8 24 | }; 25 | 26 | void set_verbosity(int v); 27 | int get_verbosity(void); 28 | } 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /mmap_allocator.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMAP_ALLOCATOR_H 2 | #define _MMAP_ALLOCATOR_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "mmap_access_mode.h" 9 | #include "mmap_exception.h" 10 | #include "mmap_file_pool.h" 11 | 12 | namespace mmap_allocator_namespace 13 | { 14 | template 15 | class mmappable_vector; 16 | 17 | template 18 | class mmap_allocator: public std::allocator 19 | { 20 | public: 21 | typedef size_t size_type; 22 | typedef off_t offset_type; 23 | typedef T* pointer; 24 | typedef const T* const_pointer; 25 | 26 | template 27 | struct rebind 28 | { 29 | typedef mmap_allocator<_Tp1> other; 30 | }; 31 | 32 | pointer allocate(size_type n, const void *hint=0) 33 | { 34 | void *the_pointer; 35 | if (get_verbosity() > 0) { 36 | fprintf(stderr, "Alloc %zd bytes.\n", n*sizeof(T)); 37 | } 38 | if (access_mode == DEFAULT_STL_ALLOCATOR) { 39 | return std::allocator::allocate(n, hint); 40 | } else { 41 | if (n == 0) { 42 | return NULL; 43 | } 44 | if (bypass_file_pool) { 45 | the_pointer = private_file.open_and_mmap_file(filename, access_mode, offset, n*sizeof(T), map_whole_file, allow_remap); 46 | } else { 47 | the_pointer = the_pool.mmap_file(filename, access_mode, offset, n*sizeof(T), map_whole_file, allow_remap); 48 | } 49 | if (the_pointer == NULL) { 50 | throw(mmap_allocator_exception("Couldn't mmap file, mmap_file returned NULL")); 51 | } 52 | if (get_verbosity() > 0) { 53 | fprintf(stderr, "pointer = %p\n", the_pointer); 54 | } 55 | 56 | return (pointer)the_pointer; 57 | } 58 | } 59 | 60 | void deallocate(pointer p, size_type n) 61 | { 62 | if (get_verbosity() > 0) { 63 | fprintf(stderr, "Dealloc %zd bytes (%p).\n", n*sizeof(T), p); 64 | } 65 | if (access_mode == DEFAULT_STL_ALLOCATOR) { 66 | std::allocator::deallocate(p, n); 67 | } else { 68 | if (n == 0) { 69 | return; 70 | } 71 | if (bypass_file_pool) { 72 | private_file.munmap_and_close_file(); 73 | } else { 74 | if (!keep_forever) { 75 | the_pool.munmap_file(filename, access_mode, offset, n*sizeof(T)); 76 | } 77 | } 78 | } 79 | } 80 | 81 | mmap_allocator() throw(): 82 | std::allocator(), 83 | filename(""), 84 | offset(0), 85 | access_mode(DEFAULT_STL_ALLOCATOR), 86 | map_whole_file(false), 87 | allow_remap(false), 88 | bypass_file_pool(false), 89 | private_file(), 90 | keep_forever(false) 91 | { } 92 | 93 | mmap_allocator(const std::allocator &a) throw(): 94 | std::allocator(a), 95 | filename(""), 96 | offset(0), 97 | access_mode(DEFAULT_STL_ALLOCATOR), 98 | map_whole_file(false), 99 | allow_remap(false), 100 | bypass_file_pool(false), 101 | private_file(), 102 | keep_forever(false) 103 | { } 104 | 105 | mmap_allocator(const mmap_allocator &a) throw(): 106 | std::allocator(a), 107 | filename(a.filename), 108 | offset(a.offset), 109 | access_mode(a.access_mode), 110 | map_whole_file(a.map_whole_file), 111 | allow_remap(a.allow_remap), 112 | bypass_file_pool(a.bypass_file_pool), 113 | private_file(a.private_file), 114 | keep_forever(a.keep_forever) 115 | { } 116 | mmap_allocator(const std::string filename_param, enum access_mode access_mode_param = READ_ONLY, offset_type offset_param = 0, int flags = 0) throw(): 117 | std::allocator(), 118 | filename(filename_param), 119 | offset(offset_param), 120 | access_mode(access_mode_param), 121 | map_whole_file((flags & MAP_WHOLE_FILE) != 0), 122 | allow_remap((flags & ALLOW_REMAP) != 0), 123 | bypass_file_pool((flags & BYPASS_FILE_POOL) != 0), 124 | private_file(), 125 | keep_forever((flags & KEEP_FOREVER) != 0) 126 | { 127 | } 128 | 129 | ~mmap_allocator() throw() { } 130 | 131 | private: 132 | friend class mmappable_vector >; 133 | 134 | std::string filename; 135 | offset_type offset; 136 | enum access_mode access_mode; 137 | bool map_whole_file; 138 | bool allow_remap; 139 | bool bypass_file_pool; 140 | mmapped_file private_file; /* used if bypass is set */ 141 | bool keep_forever; 142 | }; 143 | } 144 | 145 | #endif /* _MMAP_ALLOCATOR_H */ 146 | -------------------------------------------------------------------------------- /mmap_exception.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMAP_EXCEPTION 2 | #define _MMAP_EXCEPTION 3 | 4 | #include "mmap_access_mode.h" 5 | #include 6 | 7 | namespace mmap_allocator_namespace 8 | { 9 | class mmap_allocator_exception: public std::runtime_error { 10 | public: 11 | mmap_allocator_exception(const char *msg_param) throw(): 12 | std::runtime_error(msg_param) 13 | { 14 | if (get_verbosity() > 0) { 15 | fprintf(stderr, "Throwing exception %s\n", msg_param); 16 | } 17 | } 18 | 19 | mmap_allocator_exception(std::string msg_param) throw(): 20 | std::runtime_error(msg_param) 21 | { 22 | if (get_verbosity() > 0) { 23 | fprintf(stderr, "Throwing exception %s\n", msg_param.c_str()); 24 | } 25 | } 26 | 27 | virtual ~mmap_allocator_exception(void) throw() 28 | { 29 | } 30 | private: 31 | }; 32 | } 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /mmap_file_pool.cpp: -------------------------------------------------------------------------------- 1 | #include "mmap_file_pool.h" 2 | #include "mmap_exception.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace mmap_allocator_namespace { 12 | 13 | #ifndef MMAP_ALLOCATOR_DEBUG 14 | #define MMAP_ALLOCATOR_DEBUG 0 15 | #endif 16 | 17 | int verbosity = MMAP_ALLOCATOR_DEBUG; 18 | 19 | void set_verbosity(int v) 20 | { 21 | verbosity = v; 22 | } 23 | 24 | int get_verbosity(void) 25 | { 26 | return verbosity; 27 | } 28 | 29 | off_t filesize(int fd, std::string fname) 30 | { 31 | struct stat buf; 32 | if (fstat(fd, &buf) < 0) { 33 | if (get_verbosity() > 0) { 34 | perror("stat"); 35 | } 36 | 37 | throw mmap_allocator_exception("Cannot stat file" + fname); 38 | } 39 | 40 | return buf.st_size; 41 | } 42 | 43 | mmap_file_identifier::mmap_file_identifier(std::string fname, enum access_mode access_mode_param) 44 | { 45 | struct stat buf; 46 | if (stat(fname.c_str(), &buf) < 0) { 47 | if (get_verbosity() > 0) { 48 | perror("stat"); 49 | } 50 | throw mmap_allocator_exception("Cannot stat file "+fname); 51 | } 52 | 53 | device = buf.st_dev; 54 | inode = buf.st_ino; 55 | access_mode = access_mode_param; 56 | } 57 | 58 | mmap_file_identifier::mmap_file_identifier(const mmap_file_identifier &other) 59 | { 60 | device = other.device; 61 | inode = other.inode; 62 | access_mode = other.access_mode; 63 | } 64 | 65 | bool mmap_file_identifier::operator==(const mmap_file_identifier &other) const 66 | { 67 | return device == other.device && 68 | inode == other.inode && 69 | access_mode == other.access_mode; 70 | } 71 | 72 | bool mmap_file_identifier::operator<(const mmap_file_identifier &other) const 73 | { 74 | return inode < other.inode || 75 | (inode == other.inode && access_mode < other.access_mode) || 76 | (inode == other.inode && access_mode == other.access_mode && device < other.device); 77 | 78 | } 79 | 80 | void *mmapped_file::open_and_mmap_file(std::string filename, enum access_mode access_mode, off_t offset, size_t length, bool map_whole_file, bool allow_remap) 81 | { 82 | if (filename.c_str()[0] == '\0') { 83 | throw mmap_allocator_exception("mmap_allocator not correctly initialized: filename is empty."); 84 | } 85 | int mode; 86 | int prot; 87 | int mmap_mode = 0; 88 | off_t offset_to_map; 89 | size_t length_to_map; 90 | void *address_to_map = NULL; 91 | 92 | if (memory_area != NULL) { 93 | address_to_map = memory_area; 94 | 95 | /* do not use MAP_FIXED, since that may invalidate other memory 96 | areas in the process, such as shared libraries, which would 97 | lead to a mystic Segfault. */ 98 | 99 | } 100 | switch (access_mode) { 101 | case READ_ONLY: mode = O_RDONLY; prot = PROT_READ; mmap_mode |= MAP_SHARED; break; 102 | case READ_WRITE_SHARED: mode = O_RDWR; prot = PROT_READ | PROT_WRITE; mmap_mode |= MAP_SHARED; break; 103 | case READ_WRITE_PRIVATE: mode = O_RDONLY; prot = PROT_READ | PROT_WRITE; mmap_mode |= MAP_PRIVATE; break; 104 | default: throw mmap_allocator_exception("Internal error"); break; 105 | } 106 | 107 | if (fd == -1) { 108 | fd = open(filename.c_str(), mode); 109 | if (fd < 0) { 110 | if (get_verbosity() > 0) { 111 | perror("open"); 112 | } 113 | 114 | throw mmap_allocator_exception("Error opening file " + filename); 115 | } 116 | } 117 | if (map_whole_file) { 118 | offset_to_map = 0; 119 | length_to_map = filesize(fd, filename); 120 | } else { 121 | offset_to_map = ALIGN_TO_PAGE(offset); 122 | length_to_map = UPPER_ALIGN_TO_PAGE(length); 123 | } 124 | 125 | if (offset_to_map == offset_mapped && length_to_map == size_mapped) { 126 | reference_count++; 127 | return ((char*)memory_area)+offset-offset_mapped; 128 | } 129 | if (offset_to_map >= offset_mapped && length_to_map + offset_to_map - offset_mapped <= size_mapped) 130 | { 131 | reference_count++; 132 | return ((char*)memory_area)+offset-offset_mapped; 133 | } 134 | 135 | if (memory_area != NULL) { 136 | if (munmap(memory_area, size_mapped) < 0) { 137 | if (get_verbosity() > 0) { 138 | perror("munmap"); 139 | } 140 | throw mmap_allocator_exception("Error in munmap file "+filename); 141 | } 142 | } 143 | 144 | memory_area = mmap(address_to_map, length_to_map, prot, mmap_mode, fd, offset_to_map); 145 | if (address_to_map != NULL && !allow_remap && memory_area != MAP_FAILED && memory_area != address_to_map) { 146 | if (munmap(memory_area, length_to_map) < 0) { 147 | if (get_verbosity() > 0) { 148 | perror("munmap"); 149 | } 150 | throw mmap_allocator_exception("Error in munmap" + filename); 151 | } 152 | throw mmap_allocator_exception("Request to remap area but allow_remap is not given (remapping "+filename+")"); 153 | } 154 | 155 | if (memory_area == MAP_FAILED) { 156 | if (get_verbosity() > 0) { 157 | perror("mmap"); 158 | } 159 | throw mmap_allocator_exception("Error in mmap "+filename); 160 | } 161 | offset_mapped = offset_to_map; 162 | size_mapped = length_to_map; 163 | reference_count++; 164 | 165 | void *ret = ((char*)memory_area)+offset-offset_to_map; 166 | // assert(ret >= memory_area && ret < (char*)memory_area+size_mapped); 167 | 168 | return ret; 169 | } 170 | 171 | bool mmapped_file::munmap_and_close_file(void) 172 | { 173 | reference_count--; 174 | if (reference_count > 0) { 175 | return false; 176 | } 177 | if (munmap(memory_area, size_mapped) < 0) { 178 | if (get_verbosity() > 0) { 179 | perror("munmap"); 180 | } 181 | throw mmap_allocator_exception("Error in munmap"); 182 | } 183 | if (close(fd)) { 184 | if (get_verbosity() > 0) { 185 | perror("close"); 186 | } 187 | throw mmap_allocator_exception("Error in close"); 188 | } 189 | fd = -1; 190 | return true; 191 | } 192 | 193 | void *mmap_file_pool::mmap_file(std::string fname, enum access_mode access_mode, off_t offset, size_t length, bool map_whole_file, bool allow_remap) 194 | { 195 | mmap_file_identifier the_identifier(fname, access_mode); 196 | mmapped_file_map_t::iterator it; 197 | 198 | it = the_map.find(the_identifier); 199 | if (it != the_map.end()) { 200 | return it->second.open_and_mmap_file(fname, access_mode, offset, length, map_whole_file, allow_remap); 201 | } else { 202 | mmapped_file the_file; 203 | void *ret; 204 | 205 | ret = the_file.open_and_mmap_file(fname, access_mode, offset, length, map_whole_file, allow_remap); 206 | the_map.insert(mmapped_file_pair_t(the_identifier, the_file)); 207 | return ret; 208 | } 209 | } 210 | 211 | void mmap_file_pool::munmap_file(std::string fname, enum access_mode access_mode, off_t offset, size_t length) 212 | { 213 | mmap_file_identifier the_identifier(fname, access_mode); 214 | mmapped_file_map_t::iterator it; 215 | 216 | it = the_map.find(the_identifier); 217 | if (it != the_map.end()) { 218 | if (it->second.munmap_and_close_file()) { 219 | the_map.erase(it); 220 | } 221 | } else { 222 | throw mmap_allocator_exception("File "+fname+" not found in pool"); 223 | } 224 | } 225 | 226 | mmap_file_pool the_pool; /* TODO: move to app object */ 227 | } 228 | -------------------------------------------------------------------------------- /mmap_file_pool.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "mmap_access_mode.h" 7 | 8 | namespace mmap_allocator_namespace { 9 | class mmap_file_identifier { 10 | public: 11 | mmap_file_identifier(); 12 | mmap_file_identifier(const mmap_file_identifier &other); 13 | mmap_file_identifier(std::string fname, enum access_mode access_mode); 14 | bool operator==(const mmap_file_identifier &other) const; 15 | bool operator<(const mmap_file_identifier &other) const; 16 | 17 | private: 18 | enum access_mode access_mode; 19 | dev_t device; 20 | ino_t inode; 21 | }; 22 | 23 | 24 | class mmapped_file { 25 | public: 26 | mmapped_file(): 27 | fd(-1), 28 | memory_area(NULL), 29 | size_mapped(0), 30 | offset_mapped(0), 31 | reference_count(0) 32 | { } 33 | 34 | void *get_memory_area(void) 35 | { 36 | return memory_area; 37 | } 38 | 39 | void* open_and_mmap_file(std::string fname, enum access_mode access_mode, off_t offset, size_t length, bool map_whole_file, bool allow_remap); 40 | bool munmap_and_close_file(void); 41 | 42 | private: 43 | friend class mmap_file_pool; 44 | 45 | int fd; 46 | void *memory_area; 47 | size_t size_mapped; 48 | off_t offset_mapped; 49 | int reference_count; 50 | }; 51 | 52 | typedef std::map mmapped_file_map_t; 53 | typedef std::pair mmapped_file_pair_t; 54 | 55 | /* Singleton */ 56 | class mmap_file_pool { 57 | public: 58 | mmap_file_pool(): 59 | the_map() 60 | { } 61 | 62 | void *mmap_file(std::string fname, enum access_mode access_mode, off_t offset, size_t length, bool map_whole_file, bool allow_remap); 63 | void munmap_file(std::string fname, enum access_mode access_mode, off_t offset, size_t length); 64 | 65 | private: 66 | mmapped_file_map_t the_map; 67 | }; 68 | 69 | extern mmap_file_pool the_pool; 70 | } 71 | -------------------------------------------------------------------------------- /mmappable_vector.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMAPPABLE_VECTOR_H 2 | #define _MMAPPABLE_VECTOR_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "mmap_allocator.h" 9 | 10 | namespace mmap_allocator_namespace { 11 | template > 12 | class mmappable_vector: public std::vector { 13 | public: 14 | typedef std::vector Base; 15 | 16 | typedef typename Base::const_iterator const_iterator; 17 | typedef typename Base::iterator iterator; 18 | typedef T value_type; 19 | typedef A allocator_type; 20 | 21 | mmappable_vector(): 22 | Base() 23 | { 24 | } 25 | 26 | mmappable_vector(const mmappable_vector &other): 27 | Base(other) 28 | { 29 | } 30 | 31 | explicit mmappable_vector(size_t n): 32 | Base() 33 | { 34 | mmap_file(n); 35 | } 36 | 37 | explicit mmappable_vector(A alloc): 38 | Base(alloc) 39 | { 40 | } 41 | 42 | mmappable_vector(iterator from, iterator to): 43 | Base(from, to) 44 | { 45 | } 46 | 47 | template 48 | mmappable_vector(Iter first, Iter last, A a = A()): 49 | Base(first, last, a) 50 | { 51 | } 52 | 53 | mmappable_vector(int n, T val, A alloc): 54 | Base(n, val, alloc) 55 | { 56 | } 57 | 58 | mmappable_vector(int n, T val): 59 | Base(n, val) 60 | { 61 | } 62 | 63 | mmappable_vector(std::vector > v): 64 | std::vector >(v) 65 | { 66 | } 67 | 68 | /* Use this only when the allocator is already initialized. */ 69 | void mmap_file(size_t n) 70 | { 71 | Base::reserve(n); 72 | _M_set_finish(n); 73 | } 74 | 75 | void mmap_file(std::string filename, enum access_mode access_mode, const off_t offset, const size_t n, int flags = 0) 76 | { 77 | if (Base::size() > 0) { 78 | throw mmap_allocator_exception("Remapping currently not implemented."); 79 | } 80 | #ifdef __GNUC__ 81 | #if __GNUC__ == 3 82 | A *the_allocator = static_cast(&(this->Base::_M_impl)); 83 | #else 84 | A *the_allocator = &Base::_M_get_Tp_allocator(); 85 | #endif 86 | #else 87 | #error "Not GNU C++, please either implement me or use GCC" 88 | #endif 89 | the_allocator->filename = filename; 90 | the_allocator->offset = offset; 91 | the_allocator->access_mode = access_mode; 92 | the_allocator->map_whole_file = (flags & MAP_WHOLE_FILE) != 0; 93 | the_allocator->allow_remap = (flags & ALLOW_REMAP) != 0; 94 | the_allocator->bypass_file_pool = (flags & BYPASS_FILE_POOL) != 0; 95 | the_allocator->keep_forever = (flags & KEEP_FOREVER) != 0; 96 | 97 | mmap_file(n); 98 | } 99 | 100 | void munmap_file(void) 101 | { 102 | size_t n = Base::size(); 103 | #ifdef __GNUC__ 104 | Base::_M_deallocate(Base::_M_impl._M_start, n); 105 | Base::_M_impl._M_start = 0; 106 | Base::_M_impl._M_finish = 0; 107 | Base::_M_impl._M_end_of_storage = 0; 108 | #else 109 | #error "Not GNU C++, please either implement me or use GCC" 110 | #endif 111 | } 112 | 113 | private: 114 | void _M_set_finish(size_t n) 115 | { 116 | #ifdef __GNUC__ 117 | Base::_M_impl._M_finish = Base::_M_impl._M_start + n; 118 | #else 119 | #error "Not GNU C++, please either implement me or use GCC" 120 | #endif 121 | } 122 | }; 123 | 124 | template 125 | std::vector to_std_vector(const mmappable_vector &v) 126 | { 127 | return std::vector(v.begin(), v.end()); 128 | } 129 | 130 | template 131 | mmappable_vector to_mmappable_vector(const std::vector &v) 132 | { 133 | return mmappable_vector(v.begin(), v.end()); 134 | } 135 | } 136 | 137 | #endif /* MMAPPABLE_VECTOR_H */ 138 | -------------------------------------------------------------------------------- /test_allocator.cpp: -------------------------------------------------------------------------------- 1 | #include "mmap_allocator.h" 2 | #include "mmap_exception.h" 3 | #include "mmappable_vector.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using namespace std; 16 | using namespace mmap_allocator_namespace; 17 | 18 | #define TESTFILE "testfile" 19 | #define TESTFILE2 "testfile2" 20 | 21 | void generate_test_file(int count, const char *fname) 22 | { 23 | FILE *f; 24 | int i; 25 | 26 | f = fopen(fname, "w+"); 27 | for (i=0;i > int_vec(1024, 0, mmap_allocator("", READ_ONLY)); 93 | /* Default constructor used, allocate will fail */ 94 | assert(0); 95 | } catch (mmap_allocator_exception e) { 96 | fprintf(stderr, "Exception message (expected): %s\n", e.what()); 97 | exception_thrown = true; 98 | } 99 | assert(exception_thrown); 100 | 101 | exception_thrown = false; 102 | try { 103 | vector > int_vec_notexsting_file(1024, 0, mmap_allocator("karin", READ_ONLY)); /* no such file or directory */ 104 | assert(0); 105 | } catch (mmap_allocator_exception e) { 106 | fprintf(stderr, "Exception message (expected): %s\n", e.what()); 107 | exception_thrown = true; 108 | } 109 | assert(exception_thrown); 110 | 111 | exception_thrown = false; 112 | try { 113 | vector > int_vec_wrong_alignment_file(512, 0, mmap_allocator(TESTFILE, READ_WRITE_PRIVATE, 123)); /* wrong alignment */ 114 | /* No exception here expected */ 115 | } catch (mmap_allocator_exception &e) { 116 | fprintf(stderr, "Exception message (not expected): %s\n", e.what()); 117 | exception_thrown = true; 118 | } 119 | assert(!exception_thrown); 120 | } 121 | 122 | void test_mmap(void) 123 | { 124 | int i; 125 | 126 | fprintf(stderr, "Testing int_vec_default\n"); 127 | generate_test_file(1024, TESTFILE); 128 | mmappable_vector > int_vec_default = mmappable_vector >(mmap_allocator(TESTFILE, DEFAULT_STL_ALLOCATOR, 0)); 129 | int_vec_default.mmap_file(1024); 130 | assert(int_vec_default.size() == 1024); 131 | for (i=0;i<1024;i++) { 132 | int_vec_default[i] = i; /* no segfault */ 133 | assert(int_vec_default[i] == i); /* just to be sure */ 134 | } 135 | test_test_file(1024, false); 136 | 137 | fprintf(stderr, "Testing int_vec_rw_private\n"); 138 | generate_test_file(1024, TESTFILE); 139 | mmappable_vector > int_vec_rw_private = mmappable_vector >(mmap_allocator(TESTFILE, READ_WRITE_PRIVATE, 0)); 140 | int_vec_rw_private.mmap_file(1024); 141 | for (i=0;i<1024;i++) { 142 | assert(int_vec_rw_private[i] == i); 143 | } 144 | test_test_file(1024, false); 145 | 146 | fprintf(stderr, "Testing int_vec_ro\n"); 147 | mmappable_vector > int_vec_ro = mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 0)); 148 | int_vec_ro.mmap_file(1024); 149 | for (i=0;i<1024;i++) { 150 | assert(int_vec_ro[i] == i); 151 | } 152 | test_test_file(1024, false); 153 | 154 | fprintf(stderr, "Testing int_vec_shifted\n"); 155 | mmappable_vector > int_vec_shifted = mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, sizeof(int))); 156 | int_vec_shifted.mmap_file(1024-1); 157 | for (i=0;i<1024-1;i++) { 158 | assert(int_vec_shifted[i] == i+1); 159 | } 160 | test_test_file(1024, false); 161 | 162 | fprintf(stderr, "Testing int_vec_pointer\n"); 163 | mmappable_vector > *int_vec_pointer = new mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 0)); 164 | int_vec_pointer->mmap_file(1024); 165 | for (i=0;i<1024;i++) { 166 | assert((*int_vec_pointer)[i] == i); 167 | } 168 | test_test_file(1024, false); 169 | delete int_vec_pointer; 170 | 171 | fprintf(stderr, "Testing int_vec_initialized_private\n"); 172 | mmappable_vector > int_vec_initialized_private = mmappable_vector >(1024, 0, mmap_allocator(TESTFILE, READ_WRITE_PRIVATE, 0)); 173 | for (i=0;i<1024;i++) { 174 | assert(int_vec_initialized_private[i] == 0); 175 | } 176 | test_test_file(1024, false); 177 | 178 | fprintf(stderr, "Testing int_vec_initialized_shared\n"); 179 | mmappable_vector > int_vec_initialized_shared = mmappable_vector >(1024, 0, mmap_allocator(TESTFILE, READ_WRITE_SHARED, 0)); 180 | for (i=0;i<1024;i++) { 181 | assert(int_vec_initialized_shared[i] == 0); 182 | } 183 | test_test_file(1024, true); 184 | 185 | fprintf(stderr, "Testing int_vec_big\n"); 186 | generate_test_file(1024*1024, TESTFILE); 187 | mmappable_vector > int_vec_big = mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE | ALLOW_REMAP)); 188 | int_vec_big.mmap_file(1024*1024); 189 | for (i=0;i<1024*1024;i++) { 190 | if (int_vec_big[i] != i) { fprintf(stderr, "falsch: i=%d val=%d\n", i, int_vec_big[i]); } 191 | assert(int_vec_big[i] == i); 192 | } 193 | test_test_file(1024*1024, false); 194 | 195 | fprintf(stderr, "Testing int_vec_shifted_big\n"); 196 | generate_test_file(1024*1024, TESTFILE); 197 | mmappable_vector > int_vec_shifted_big = mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, sizeof(i), MAP_WHOLE_FILE | ALLOW_REMAP)); 198 | int_vec_shifted_big.mmap_file(1024*1024-1); 199 | for (i=0;i<1024*1024-1;i++) { 200 | assert(int_vec_shifted_big[i] == i+1); 201 | } 202 | test_test_file(1024*1024, false); 203 | 204 | fprintf(stderr, "Testing int_vec_big_minus_one\n"); 205 | generate_test_file(1024*1024-1, TESTFILE); 206 | mmappable_vector > int_vec_big_minus_one = mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE | ALLOW_REMAP)); 207 | int_vec_big_minus_one.mmap_file(1024*1024-1); 208 | for (i=0;i<1024*1024-1;i++) { 209 | assert(int_vec_big_minus_one[i] == i); 210 | } 211 | test_test_file(1024*1024-1, false); 212 | } 213 | 214 | void test_conversion(void) 215 | { 216 | mmappable_vector mmap_vector; 217 | vector std_vector; 218 | mmappable_vector mmap_vector2; 219 | int i; 220 | 221 | fprintf(stderr, "Testing conversion between STL vector and mmap vector.\n"); 222 | generate_test_file(1024, TESTFILE); 223 | mmap_vector.mmap_file(TESTFILE, READ_ONLY, 0, 1024); 224 | for (i=0;i<1024;i++) { 225 | assert(mmap_vector[i] == i); 226 | } 227 | 228 | std_vector = to_std_vector(mmap_vector); 229 | for (i=0;i<1024;i++) { 230 | assert(std_vector[i] == i); 231 | } 232 | for (i=0;i<1024;i++) { 233 | std_vector[i] *= 2; 234 | } 235 | mmap_vector2 = to_mmappable_vector(std_vector); 236 | for (i=0;i<1024;i++) { 237 | assert(mmap_vector2[i] == i*2); 238 | } 239 | } 240 | 241 | 242 | void test_mmap_file_pool(void) 243 | { 244 | generate_test_file(1024, TESTFILE); 245 | int *f = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 1024, false, false); 246 | int *f2 = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 1024, false, false); 247 | int i; 248 | 249 | assert(f == f2); 250 | 251 | for (i=0;i<1024;i++) { 252 | assert(f[i] == i); 253 | } 254 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 1024); 255 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 1024); 256 | } 257 | 258 | void test_mapping_smaller_area(void) 259 | { 260 | fprintf(stderr, "Testing mapping of areas that fit in already mapped areas\n"); 261 | generate_test_file(2048, TESTFILE); 262 | 263 | int *f = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 8192, false, false); 264 | int *first_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 4096, false, false); 265 | int *second_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 4096, 4096, false, false); 266 | 267 | assert(f == first_page); 268 | assert(f+1024 == second_page); 269 | 270 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 8192); 271 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 4096); 272 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 4096, 4096); 273 | } 274 | 275 | 276 | void test_mapping_smaller_area_whole_file_flag(void) 277 | { 278 | fprintf(stderr, "Testing whole file flag\n"); 279 | generate_test_file(2048, TESTFILE); 280 | 281 | int *f = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 1, true, false); 282 | int *first_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 4096, false, false); 283 | int *second_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 4096, 4096, false, false); 284 | 285 | assert(f == first_page); 286 | assert(f+1024 == second_page); 287 | 288 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 8192); 289 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 4096); 290 | the_pool.munmap_file(string(TESTFILE), READ_ONLY, 4096, 4096); 291 | } 292 | 293 | 294 | void test_mapping_smaller_area_whole_file_flag_allocator(void) 295 | { 296 | int i; 297 | 298 | fprintf(stderr, "Testing whole file flag via allocator\n"); 299 | generate_test_file(2048, TESTFILE); 300 | 301 | fprintf(stderr, "Testing int_vec_ro\n"); 302 | mmappable_vector > int_vec_ro = mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE)); 303 | int_vec_ro.mmap_file(1024); 304 | for (i=0;i<1024;i++) { 305 | assert(int_vec_ro[i] == i); 306 | } 307 | 308 | mmappable_vector > int_vec_ro_second_page = mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 4096, MAP_WHOLE_FILE)); 309 | int_vec_ro_second_page.mmap_file(1024); 310 | for (i=0;i<1024;i++) { 311 | assert(int_vec_ro_second_page[i] == i+1024); 312 | } 313 | } 314 | 315 | 316 | void test_mapping_smaller_area_whole_file_flag_allocator_deleting_first_vec(void) 317 | { 318 | int i; 319 | 320 | fprintf(stderr, "Testing whole file flag via allocator, deleting first mapping before allocating second page\n"); 321 | generate_test_file(2048, TESTFILE); 322 | 323 | fprintf(stderr, "Testing int_vec_ro\n"); 324 | mmappable_vector > *int_vec_ro_p = new mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE)); 325 | int_vec_ro_p->mmap_file(1024); 326 | for (i=0;i<1024;i++) { 327 | assert((*int_vec_ro_p)[i] == i); 328 | } 329 | delete int_vec_ro_p; 330 | 331 | mmappable_vector > *int_vec_ro_second_page_p = new mmappable_vector >(mmap_allocator(TESTFILE, READ_ONLY, 4096, MAP_WHOLE_FILE)); 332 | int_vec_ro_second_page_p->mmap_file(1024); 333 | for (i=0;i<1024;i++) { 334 | assert((*int_vec_ro_second_page_p)[i] == i+1024); 335 | } 336 | delete int_vec_ro_second_page_p; 337 | } 338 | 339 | 340 | void test_new_interface(void) 341 | { 342 | int i; 343 | 344 | fprintf(stderr, "Testing new interface\n"); 345 | 346 | generate_test_file(1024, TESTFILE); 347 | 348 | mmappable_vector vec; 349 | vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024); 350 | 351 | for (i=0;i<1024;i++) { 352 | assert(vec[i] == i); 353 | } 354 | try { 355 | vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024); 356 | assert(0); 357 | } catch (mmap_allocator_exception e) { 358 | fprintf(stderr, "Exception message (expected): %s\n", e.what()); 359 | } 360 | vec.munmap_file(); 361 | 362 | generate_test_file(2048, TESTFILE); 363 | vec.mmap_file(TESTFILE, READ_ONLY, 4096, 1024); 364 | for (i=0;i<1024;i++) { 365 | assert(vec[i] == i+1024); 366 | } 367 | } 368 | 369 | void test_cache_bug(void) 370 | { 371 | mmappable_vector vec; 372 | int i; 373 | 374 | fprintf(stderr, "Testing if wrong offset bug in pool is fixed.\n"); 375 | generate_test_file(2048, TESTFILE); 376 | vec.mmap_file(TESTFILE, READ_ONLY, 4096, 1024); 377 | 378 | for (i=0;i<1024;i++) { 379 | assert(vec[i] == i+1024); 380 | } 381 | } 382 | 383 | void test_private_file_pool(void) 384 | { 385 | mmappable_vector vec; 386 | mmappable_vector vec2; 387 | int i; 388 | 389 | fprintf(stderr, "Testing if bypass_file_pool works.\n"); 390 | generate_test_file(1024, TESTFILE); 391 | vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, BYPASS_FILE_POOL); 392 | for (i=0;i<1024;i++) { 393 | assert(vec[i] == i); 394 | } 395 | 396 | vec2.mmap_file(TESTFILE, READ_ONLY, 0, 1024, BYPASS_FILE_POOL); 397 | for (i=0;i<1024;i++) { 398 | assert(vec[i] == i); 399 | } 400 | assert(&vec[0] != &vec2[0]); 401 | } 402 | 403 | #define FILESIZE (1024*1024*16) 404 | 405 | void read_large_file(enum access_mode mode) 406 | { 407 | struct timeval t, t2; 408 | mmappable_vector vec; 409 | int i; 410 | 411 | gettimeofday(&t, NULL); 412 | 413 | vec.mmap_file(TESTFILE, mode, 0, FILESIZE); 414 | for (i=0;i vec1, vec2, vec3, vec4; 436 | 437 | fprintf(stderr, "Testing multiple open (you need to strace this).\n"); 438 | vec1.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE); 439 | vec2.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE); 440 | vec3.mmap_file(TESTFILE2, READ_ONLY, 0, 1024, MAP_WHOLE_FILE); 441 | vec4.mmap_file(TESTFILE2, READ_ONLY, 0, 1024, MAP_WHOLE_FILE); 442 | } 443 | 444 | void test_keep_forever(void) 445 | { 446 | generate_test_file(1024, TESTFILE); 447 | mmappable_vector vec1, vec2, vec3, vec4; 448 | 449 | fprintf(stderr, "Testing multiple open (you need to strace this).\n"); 450 | { 451 | mmappable_vector vec; 452 | fprintf(stderr, "Testing mapping without KEEP_FOREVER (you need to strace this: there should be a close).\n"); 453 | vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE); 454 | } 455 | { 456 | mmappable_vector vec; 457 | fprintf(stderr, "Testing mapping without KEEP_FOREVER (you need to strace this: the file should be reopened).\n"); 458 | vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE); 459 | } 460 | { 461 | mmappable_vector vec; 462 | fprintf(stderr, "Testing mapping with KEEP_FOREVER (you need to strace this: there should be NO close).\n"); 463 | vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE | KEEP_FOREVER); 464 | } 465 | { 466 | mmappable_vector vec; 467 | fprintf(stderr, "Testing mapping with KEEP_FOREVER (you need to strace this: the file shouldn't be reopened).\n"); 468 | vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE | KEEP_FOREVER); 469 | } 470 | } 471 | 472 | 473 | void test_allocate_0_bytes(void) /* shouldn't segfault */ 474 | { 475 | fprintf(stderr, "Testing vectors of mmappable_vectors.\n"); 476 | 477 | vector > vecs; 478 | vecs.resize(2); 479 | for (int i=0; i<2; i++) { 480 | vecs[i].mmap_file(TESTFILE, READ_ONLY, 0, 1024, 0); 481 | for (int j=0;j<1024;j++) { 482 | assert(vecs[i][j] == j); 483 | } 484 | } 485 | } 486 | 487 | 488 | int main(int argc, char ** argv) 489 | { 490 | set_verbosity(1); 491 | 492 | test_page_align_macros(); 493 | test_throw_catch(); 494 | test_exceptions(); 495 | test_mmap_file_pool(); 496 | test_mmap(); 497 | test_conversion(); 498 | test_cache_bug(); 499 | test_mapping_smaller_area(); 500 | test_mapping_smaller_area_whole_file_flag(); 501 | test_mapping_smaller_area_whole_file_flag_allocator(); 502 | test_mapping_smaller_area_whole_file_flag_allocator_deleting_first_vec(); 503 | test_new_interface(); 504 | test_private_file_pool(); 505 | test_large_file(); 506 | test_multiple_open(); 507 | test_keep_forever(); 508 | test_allocate_0_bytes(); 509 | } 510 | -------------------------------------------------------------------------------- /test_mmap_fixed.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | void generate_test_file(int count) 9 | { 10 | FILE *f; 11 | int i; 12 | 13 | f = fopen("testfile", "w+"); 14 | for (i=0;i=0); 28 | 29 | area_short = mmap(0, 4096, PROT_READ, MAP_PRIVATE, fd, 0); 30 | assert(area_short != MAP_FAILED); 31 | 32 | close(fd); 33 | generate_test_file(1024*1024); 34 | fd = open("testfile", O_RDONLY); 35 | assert(fd>=0); 36 | 37 | area_long = mmap(area_short, 4096*1024, PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); 38 | assert(area_long != MAP_FAILED); 39 | assert(area_short == area_long); 40 | } 41 | 42 | int main(int argc, char ** argv) 43 | { 44 | test_mmap_directly(); 45 | /* Crashes on shutdown for libc 2.5 and kernel 2.6.18 (CentOS 5) */ 46 | } 47 | --------------------------------------------------------------------------------