├── LICENSE
├── readme.md
├── mywcl.cpp
├── MemoryMapped.h
└── MemoryMapped.cpp


/LICENSE:
--------------------------------------------------------------------------------
 1 | zlib License
 2 | 
 3 | Copyright (c) 2013 Stephan Brumme
 4 | 
 5 | This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.
 6 | Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:
 7 | 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software.
 8 |    If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
 9 | 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
10 | 3. This notice may not be removed or altered from any source distribution.
11 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | This is a mirror of my library hosted at https://create.stephan-brumme.com/portable-memory-mapping/
 2 | 
 3 | # Portable Memory Mapping C++ Class
 4 | 
 5 | Memory mapping is one of the nicest features of modern operating systems:
 6 | after opening a file in memory-mapped mode you can treat the file as a large chunk of memory and use plain pointers.
 7 | The operating system takes care of loading the data on demand (!) into memory - utilizing caches, of course.
 8 | 
 9 | When using my C++ class `MemoryMapped` it's really easy:
10 | ``` cpp
11 | // open file
12 | MemoryMapped data("myfile.txt");
13 | 
14 | // read byte at file offset 2345
15 | unsigned char a = data[2345];
16 | 
17 | // or if you prefer pointers
18 | const short* raw = (const short*) data.getData();
19 | short        b   = raw[300];
20 | ```
21 | 
22 | Windows is completely different from Linux when it comes to opening a file in memory-mapped mode.
23 | The class `MemoryMapped` hides all the OS specific stuff in only two files: [MemoryMapped.h](MemoryMapped.h) and [MemoryMapped.cpp](MemoryMapped.cpp).
24 | They compile without any warning with GCC and Visual C++.
25 | 


--------------------------------------------------------------------------------
/mywcl.cpp:
--------------------------------------------------------------------------------
 1 | // //////////////////////////////////////////////////////////
 2 | // mywcl.cpp
 3 | // Copyright (c) 2013 Stephan Brumme. All rights reserved.
 4 | //
 5 | 
 6 | // g++ MemoryMapped.cpp mywcl.cpp -o mywcl -O3 -fopenmp
 7 | 
 8 | #include "MemoryMapped.h"
 9 | #include <cstdio>
10 | 
11 | int main(int argc, char* argv[])
12 | {
13 |   // syntax check
14 |   if (argc > 2)
15 |   {
16 |     printf("Syntax: ./mywcl filename\n");
17 |     return -1;
18 |   }
19 | 
20 |   // map file to memory
21 |   MemoryMapped data(argv[1], MemoryMapped::WholeFile, MemoryMapped::SequentialScan);
22 |   if (!data.isValid())
23 |   {
24 |     printf("File not found\n");
25 |     return -2;
26 |   }
27 | 
28 |   // raw pointer to mapped memory
29 |   char* buffer = (char*)data.getData();
30 |   // store result here
31 |   uint64_t numLines = 0;
32 | 
33 |   // OpenMP spreads work across CPU cores
34 | #pragma omp parallel for reduction(+:numLines)
35 |   for (uint64_t i = 0; i < data.size(); i++)
36 |     numLines += (buffer[i] == '\n');
37 | 
38 |   // show result
39 | #ifdef _MSC_VER
40 |   printf("%I64d\n", numLines);
41 | #else
42 |   printf("%lld\n",  numLines);
43 | #endif
44 |   return 0;
45 | }
46 | 


--------------------------------------------------------------------------------
/MemoryMapped.h:
--------------------------------------------------------------------------------
  1 | // //////////////////////////////////////////////////////////
  2 | // MemoryMapped.h
  3 | // Copyright (c) 2013 Stephan Brumme. All rights reserved.
  4 | // see http://create.stephan-brumme.com/disclaimer.html
  5 | //
  6 | 
  7 | #pragma once
  8 | 
  9 | // define fixed size integer types
 10 | #ifdef _MSC_VER
 11 | typedef unsigned __int64 uint64_t;
 12 | #else
 13 | #include <stdint.h>
 14 | #endif
 15 | 
 16 | #include <string>
 17 | 
 18 | 
 19 | /// Portable read-only memory mapping (Windows and Linux)
 20 | /** Filesize limited by size_t, usually 2^32 or 2^64 */
 21 | class MemoryMapped
 22 | {
 23 | public:
 24 |   /// tweak performance
 25 |   enum CacheHint
 26 |   {
 27 |     Normal,         ///< good overall performance
 28 |     SequentialScan, ///< read file only once with few seeks
 29 |     RandomAccess    ///< jump around
 30 |   };
 31 | 
 32 |   /// how much should be mappend
 33 |   enum MapRange
 34 |   {
 35 |     WholeFile = 0   ///< everything ... be careful when file is larger than memory
 36 |   };
 37 | 
 38 |   /// do nothing, must use open()
 39 |   MemoryMapped();
 40 |   /// open file, mappedBytes = 0 maps the whole file
 41 |   MemoryMapped(const std::string& filename, size_t mappedBytes = WholeFile, CacheHint hint = Normal);
 42 |   /// close file (see close() )
 43 |   ~MemoryMapped();
 44 | 
 45 |   /// open file, mappedBytes = 0 maps the whole file
 46 |   bool open(const std::string& filename, size_t mappedBytes = WholeFile, CacheHint hint = Normal);
 47 |   /// close file
 48 |   void close();
 49 | 
 50 |   /// access position, no range checking (faster)
 51 |   unsigned char operator[](size_t offset) const;
 52 |   /// access position, including range checking
 53 |   unsigned char at        (size_t offset) const;
 54 | 
 55 |   /// raw access
 56 |   const unsigned char* getData() const;
 57 | 
 58 |   /// true, if file successfully opened
 59 |   bool isValid() const;
 60 | 
 61 |   /// get file size
 62 |   uint64_t size() const;
 63 |   /// get number of actually mapped bytes
 64 |   size_t   mappedSize() const;
 65 | 
 66 |   /// replace mapping by a new one of the same file, offset MUST be a multiple of the page size
 67 |   bool remap(uint64_t offset, size_t mappedBytes);
 68 | 
 69 | private:
 70 |   /// don't copy object
 71 |   MemoryMapped(const MemoryMapped&);
 72 |   /// don't copy object
 73 |   MemoryMapped& operator=(const MemoryMapped&);
 74 | 
 75 |   /// get OS page size (for remap)
 76 |   static int getpagesize();
 77 | 
 78 |   /// file name
 79 |   std::string _filename;
 80 |   /// file size
 81 |   uint64_t    _filesize;
 82 |   /// caching strategy
 83 |   CacheHint   _hint;
 84 |   /// mapped size
 85 |   size_t      _mappedBytes;
 86 | 
 87 |   /// define handle
 88 | #ifdef _MSC_VER
 89 |   typedef void* FileHandle;
 90 |   /// Windows handle to memory mapping of _file
 91 |   void*       _mappedFile;
 92 | #else
 93 |   typedef int   FileHandle;
 94 | #endif
 95 | 
 96 |   /// file handle
 97 |   FileHandle  _file;
 98 |   /// pointer to the file contents mapped into memory
 99 |   void*       _mappedView;
100 | };
101 | 


--------------------------------------------------------------------------------
/MemoryMapped.cpp:
--------------------------------------------------------------------------------
  1 | // //////////////////////////////////////////////////////////
  2 | // MemoryMapped.cpp
  3 | // Copyright (c) 2013 Stephan Brumme. All rights reserved.
  4 | // see http://create.stephan-brumme.com/disclaimer.html
  5 | //
  6 | 
  7 | #include "MemoryMapped.h"
  8 | 
  9 | #include <stdexcept>
 10 | #include <cstdio>
 11 | 
 12 | // OS-specific
 13 | #ifdef _MSC_VER
 14 | // Windows
 15 | #include <windows.h>
 16 | #else
 17 | // Linux
 18 | // enable large file support on 32 bit systems
 19 | #ifndef _LARGEFILE64_SOURCE
 20 | #define _LARGEFILE64_SOURCE
 21 | #endif
 22 | #ifdef  _FILE_OFFSET_BITS
 23 | #undef  _FILE_OFFSET_BITS
 24 | #endif
 25 | #define _FILE_OFFSET_BITS 64
 26 | // and include needed headers
 27 | #include <sys/stat.h>
 28 | #include <sys/mman.h>
 29 | #include <fcntl.h>
 30 | #include <errno.h>
 31 | #include <unistd.h>
 32 | #endif
 33 | 
 34 | 
 35 | /// do nothing, must use open()
 36 | MemoryMapped::MemoryMapped()
 37 | : _filename   (),
 38 |   _filesize   (0),
 39 |   _hint       (Normal),
 40 |   _mappedBytes(0),
 41 |   _file       (0),
 42 | #ifdef _MSC_VER
 43 |   _mappedFile (NULL),
 44 | #endif
 45 |   _mappedView (NULL)
 46 | {
 47 | }
 48 | 
 49 | 
 50 | /// open file, mappedBytes = 0 maps the whole file
 51 | MemoryMapped::MemoryMapped(const std::string& filename, size_t mappedBytes, CacheHint hint)
 52 | : _filename   (filename),
 53 |   _filesize   (0),
 54 |   _hint       (hint),
 55 |   _mappedBytes(mappedBytes),
 56 |   _file       (0),
 57 | #ifdef _MSC_VER
 58 |   _mappedFile (NULL),
 59 | #endif
 60 |   _mappedView (NULL)
 61 | {
 62 |   open(filename, mappedBytes, hint);
 63 | }
 64 | 
 65 | 
 66 | /// close file (see close() )
 67 | MemoryMapped::~MemoryMapped()
 68 | {
 69 |   close();
 70 | }
 71 | 
 72 | 
 73 | /// open file
 74 | bool MemoryMapped::open(const std::string& filename, size_t mappedBytes, CacheHint hint)
 75 | {
 76 |   // already open ?
 77 |   if (isValid())
 78 |     return false;
 79 | 
 80 |   _file       = 0;
 81 |   _filesize   = 0;
 82 |   _hint       = hint;
 83 | #ifdef _MSC_VER
 84 |   _mappedFile = NULL;
 85 | #endif
 86 |   _mappedView = NULL;
 87 | 
 88 | #ifdef _MSC_VER
 89 |   // Windows
 90 | 
 91 |   DWORD winHint = 0;
 92 |   switch (_hint)
 93 |   {
 94 |   case Normal:         winHint = FILE_ATTRIBUTE_NORMAL;     break;
 95 |   case SequentialScan: winHint = FILE_FLAG_SEQUENTIAL_SCAN; break;
 96 |   case RandomAccess:   winHint = FILE_FLAG_RANDOM_ACCESS;   break;
 97 |   default: break;
 98 |   }
 99 | 
100 |   // open file
101 |   _file = ::CreateFileA(filename.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, winHint, NULL);
102 |   if (!_file)
103 |     return false;
104 | 
105 |   // file size
106 |   LARGE_INTEGER result;
107 |   if (!GetFileSizeEx(_file, &result))
108 |     return false;
109 |   _filesize = static_cast<uint64_t>(result.QuadPart);
110 | 
111 |   // convert to mapped mode
112 |   _mappedFile = ::CreateFileMapping(_file, NULL, PAGE_READONLY, 0, 0, NULL);
113 |   if (!_mappedFile)
114 |     return false;
115 | 
116 | #else
117 | 
118 |   // Linux
119 | 
120 |   // open file
121 |   _file = ::open(filename.c_str(), O_RDONLY | O_LARGEFILE);
122 |   if (_file == -1)
123 |   {
124 |     _file = 0;
125 |     return false;
126 |   }
127 | 
128 |   // file size
129 |   struct stat64 statInfo;
130 |   if (fstat64(_file, &statInfo) < 0)
131 |     return false;
132 | 
133 |   _filesize = statInfo.st_size;
134 | #endif
135 | 
136 |   // initial mapping
137 |   remap(0, mappedBytes);
138 | 
139 |   if (!_mappedView)
140 |     return false;
141 | 
142 |   // everything's fine
143 |   return true;
144 | }
145 | 
146 | 
147 | /// close file
148 | void MemoryMapped::close()
149 | {
150 |   // kill pointer
151 |   if (_mappedView)
152 |   {
153 | #ifdef _MSC_VER
154 |     ::UnmapViewOfFile(_mappedView);
155 | #else
156 |     ::munmap(_mappedView, _filesize);
157 | #endif
158 |     _mappedView = NULL;
159 |   }
160 | 
161 | #ifdef _MSC_VER
162 |   if (_mappedFile)
163 |   {
164 |     ::CloseHandle(_mappedFile);
165 |     _mappedFile = NULL;
166 |   }
167 | #endif
168 | 
169 |   // close underlying file
170 |   if (_file)
171 |   {
172 | #ifdef _MSC_VER
173 |     ::CloseHandle(_file);
174 | #else
175 |     ::close(_file);
176 | #endif
177 |     _file = 0;
178 |   }
179 | 
180 |   _filesize = 0;
181 | }
182 | 
183 | 
184 | /// access position, no range checking (faster)
185 | unsigned char MemoryMapped::operator[](size_t offset) const
186 | {
187 |   return ((unsigned char*)_mappedView)[offset];
188 | }
189 | 
190 | 
191 | /// access position, including range checking
192 | unsigned char MemoryMapped::at(size_t offset) const
193 | {
194 |   // checks
195 |   if (!_mappedView)
196 |     throw std::invalid_argument("No view mapped");
197 |   if (offset >= _filesize)
198 |     throw std::out_of_range("View is not large enough");
199 | 
200 |   return operator[](offset);
201 | }
202 | 
203 | 
204 | /// raw access
205 | const unsigned char* MemoryMapped::getData() const
206 | {
207 |   return (const unsigned char*)_mappedView;
208 | }
209 | 
210 | 
211 | /// true, if file successfully opened
212 | bool MemoryMapped::isValid() const
213 | {
214 |   return _mappedView != NULL;
215 | }
216 | 
217 | 
218 | /// get file size
219 | uint64_t MemoryMapped::size() const
220 | {
221 |   return _filesize;
222 | }
223 | 
224 | 
225 | /// get number of actually mapped bytes
226 | size_t MemoryMapped::mappedSize() const
227 | {
228 |   return _mappedBytes;
229 | }
230 | 
231 | 
232 | /// replace mapping by a new one of the same file, offset MUST be a multiple of the page size
233 | bool MemoryMapped::remap(uint64_t offset, size_t mappedBytes)
234 | {
235 |   if (!_file)
236 |     return false;
237 | 
238 |   if (mappedBytes == WholeFile)
239 |     mappedBytes = _filesize;
240 | 
241 |   // close old mapping
242 |   if (_mappedView)
243 |   {
244 | #ifdef _MSC_VER
245 |     ::UnmapViewOfFile(_mappedView);
246 | #else
247 |     ::munmap(_mappedView, _mappedBytes);
248 | #endif
249 |     _mappedView = NULL;
250 |   }
251 | 
252 |   // don't go further than end of file
253 |   if (offset > _filesize)
254 |     return false;
255 |   if (offset + mappedBytes > _filesize)
256 |     mappedBytes = size_t(_filesize - offset);
257 | 
258 | #ifdef _MSC_VER
259 |   // Windows
260 | 
261 |   DWORD offsetLow  = DWORD(offset & 0xFFFFFFFF);
262 |   DWORD offsetHigh = DWORD(offset >> 32);
263 |   _mappedBytes = mappedBytes;
264 | 
265 |   // get memory address
266 |   _mappedView = ::MapViewOfFile(_mappedFile, FILE_MAP_READ, offsetHigh, offsetLow, mappedBytes);
267 | 
268 |   if (_mappedView == NULL)
269 |   {
270 |     _mappedBytes = 0;
271 |     _mappedView  = NULL;
272 |     return false;
273 |   }
274 | 
275 |   return true;
276 | 
277 | #else
278 | 
279 |   // Linux
280 |   // new mapping
281 |   _mappedView = ::mmap64(NULL, mappedBytes, PROT_READ, MAP_SHARED, _file, offset);
282 |   if (_mappedView == MAP_FAILED)
283 |   {
284 |     _mappedBytes = 0;
285 |     _mappedView  = NULL;
286 |     return false;
287 |   }
288 | 
289 |   _mappedBytes = mappedBytes;
290 | 
291 |   // tweak performance
292 |   int linuxHint = 0;
293 |   switch (_hint)
294 |   {
295 |   case Normal:         linuxHint = MADV_NORMAL;     break;
296 |   case SequentialScan: linuxHint = MADV_SEQUENTIAL; break;
297 |   case RandomAccess:   linuxHint = MADV_RANDOM;     break;
298 |   default: break;
299 |   }
300 |   // assume that file will be accessed soon
301 |   //linuxHint |= MADV_WILLNEED;
302 |   // assume that file will be large
303 |   //linuxHint |= MADV_HUGEPAGE;
304 | 
305 |   ::madvise(_mappedView, _mappedBytes, linuxHint);
306 | 
307 |   return true;
308 | #endif
309 | }
310 | 
311 | 
312 | /// get OS page size (for remap)
313 | int MemoryMapped::getpagesize()
314 | {
315 | #ifdef _MSC_VER
316 |   SYSTEM_INFO sysInfo;
317 |   GetSystemInfo(&sysInfo);
318 |   return sysInfo.dwAllocationGranularity;
319 | #else
320 |   return sysconf(_SC_PAGESIZE); //::getpagesize();
321 | #endif
322 | }
323 | 


--------------------------------------------------------------------------------