├── go.mod ├── go.sum ├── README.md ├── mmap_test.go └── mmap.go /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tidwall/mmap 2 | 3 | go 1.15 4 | 5 | require github.com/edsrzf/mmap-go v1.1.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= 2 | github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= 3 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 4 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mmap 2 | 3 | Load file-backed memory. 4 | Uses [edsrzf/mmap-go](https://github.com/edsrzf/mmap-go) under the hood. 5 | 6 | ## Installing 7 | 8 | ``` 9 | go get -u github.com/tidwall/mmap 10 | ``` 11 | 12 | ## Using 13 | 14 | Load a bigole file into a byte slice. This happens pretty much instantly even 15 | if your file is many GBs. 16 | 17 | ```go 18 | data, err := mmap.Open("my-big-file.txt", false) 19 | if err != nil { 20 | panic(err) 21 | } 22 | ``` 23 | 24 | Now you can read the `data` slice like any other Go slice. 25 | 26 | Make sure to release the data when your done. 27 | 28 | ```go 29 | mmap.Close(data) 30 | ``` 31 | 32 | Don't read the `data` after closing otherwise your f*cked. 33 | 34 | That's all, bye now 35 | -------------------------------------------------------------------------------- /mmap_test.go: -------------------------------------------------------------------------------- 1 | package mmap 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestMMap(t *testing.T) { 9 | defer os.RemoveAll("test.dat") 10 | err := os.WriteFile("test.dat", []byte("hello world"), 0666) 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | data, err := Open("test.dat", false) 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | if string(data) != "hello world" { 19 | t.Fatalf("expected '%s' got '%s'", "hello world", string(data)) 20 | } 21 | if err := Close(data); err != nil { 22 | t.Fatal(err) 23 | } 24 | data, err = Open("test.dat", true) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | data[0] = 'j' 29 | copy(data[6:], "earth") 30 | if string(data) != "jello earth" { 31 | t.Fatalf("expected '%s' got '%s'", "jello world", string(data)) 32 | } 33 | if err := Close(data); err != nil { 34 | t.Fatal(err) 35 | } 36 | data, err = Open("test.dat", false) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | if string(data) != "jello earth" { 41 | t.Fatalf("expected '%s' got '%s'", "jello earth", string(data)) 42 | } 43 | if err := Close(data); err != nil { 44 | t.Fatal(err) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /mmap.go: -------------------------------------------------------------------------------- 1 | package mmap 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "sync" 7 | "unsafe" 8 | 9 | "github.com/edsrzf/mmap-go" 10 | ) 11 | 12 | type mapContext struct { 13 | f *os.File 14 | opened bool 15 | } 16 | 17 | var mmapMu sync.Mutex 18 | var mmapFiles map[unsafe.Pointer]mapContext 19 | 20 | // MapFile maps an opened file to a byte slice of data. 21 | func MapFile(f *os.File, writable bool) (data []byte, err error) { 22 | prot := mmap.RDONLY 23 | if writable { 24 | prot = mmap.RDWR 25 | } 26 | fi, err := f.Stat() 27 | if err != nil { 28 | return nil, err 29 | } 30 | if fi.Size() == 0 { 31 | return nil, nil 32 | } 33 | m, err := mmap.Map(f, prot, 0) 34 | if err != nil { 35 | return nil, err 36 | } 37 | if len(m) == 0 { 38 | // Empty file. Release map 39 | m.Unmap() 40 | f.Close() 41 | return nil, nil 42 | } 43 | if runtime.GOOS == "windows" { 44 | // Keep track of the file. 45 | mmapMu.Lock() 46 | if mmapFiles == nil { 47 | mmapFiles = make(map[unsafe.Pointer]mapContext) 48 | } 49 | mmapFiles[unsafe.Pointer(&m[0])] = mapContext{f, false} 50 | mmapMu.Unlock() 51 | } 52 | return []byte(m), nil 53 | } 54 | 55 | // Open will mmap a file to a byte slice of data. 56 | func Open(path string, writable bool) (data []byte, err error) { 57 | flag := os.O_RDONLY 58 | if writable { 59 | flag = os.O_RDWR 60 | } 61 | f, err := os.OpenFile(path, flag, 0) 62 | if err != nil { 63 | return nil, err 64 | } 65 | m, err := MapFile(f, writable) 66 | if err != nil { 67 | f.Close() 68 | return nil, err 69 | } 70 | if len(m) == 0 { 71 | // Empty file. Release map 72 | Close(m) 73 | f.Close() 74 | return nil, nil 75 | } 76 | if runtime.GOOS == "windows" { 77 | mmapMu.Lock() 78 | ctx := mmapFiles[unsafe.Pointer(&m[0])] 79 | ctx.opened = true 80 | mmapFiles[unsafe.Pointer(&m[0])] = ctx 81 | mmapMu.Unlock() 82 | } else { 83 | // Allowed to close the file. 84 | f.Close() 85 | } 86 | return []byte(m), nil 87 | } 88 | 89 | // Close releases the data. 90 | func Close(data []byte) error { 91 | if len(data) == 0 { 92 | return nil 93 | } 94 | if runtime.GOOS == "windows" { 95 | // Close file first 96 | var ctx mapContext 97 | var ok bool 98 | mmapMu.Lock() 99 | ctx, ok = mmapFiles[unsafe.Pointer(&data[0])] 100 | if ok { 101 | delete(mmapFiles, unsafe.Pointer(&data[0])) 102 | } 103 | mmapMu.Unlock() 104 | if ok && ctx.opened { 105 | ctx.f.Close() 106 | } 107 | } 108 | m := mmap.MMap(data) 109 | return m.Unmap() 110 | } 111 | 112 | // Create a new mmap file with the provided size 113 | func Create(path string, size int) ([]byte, error) { 114 | f, err := os.Create(path) 115 | if err != nil { 116 | return nil, err 117 | } 118 | if _, err := f.WriteAt([]byte{0}, int64(size)-1); err != nil { 119 | return nil, err 120 | } 121 | if err := f.Close(); err != nil { 122 | return nil, err 123 | } 124 | return Open(path, true) 125 | } 126 | --------------------------------------------------------------------------------