├── AUTHORS
├── CONTRIBUTORS
├── LICENSE
├── LICENSE-MMAP-GO
├── Makefile
├── README.md
├── all_test.go
├── memory.go
├── memory32.go
├── memory64.go
├── mmap_unix.go
├── mmap_windows.go
├── trace_disabled.go
└── trace_enabled.go


/AUTHORS:
--------------------------------------------------------------------------------
 1 | # This file lists authors for copyright purposes.  This file is distinct from
 2 | # the CONTRIBUTORS files.  See the latter for an explanation.
 3 | #
 4 | # Names should be added to this file as:
 5 | #     Name or Organization <email address>
 6 | #
 7 | # The email address is not required for organizations.
 8 | #
 9 | # Please keep the list sorted.
10 | 
11 | Jan Mercl <0xjnml@gmail.com>
12 | Steffen Butzer <steffen(dot)butzer@outlook.com>
13 | 


--------------------------------------------------------------------------------
/CONTRIBUTORS:
--------------------------------------------------------------------------------
 1 | # This file lists people who contributed code to this repository.  The AUTHORS
 2 | # file lists the copyright holders; this file lists people.
 3 | #
 4 | # Names should be added to this file like so:
 5 | #     Name <email address>
 6 | #
 7 | # Please keep the list sorted.
 8 | 
 9 | Jan Mercl <0xjnml@gmail.com>
10 | Steffen Butzer <steffen(dot)butzer@outlook.com>
11 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2017 The Memory Authors. All rights reserved.
 2 | 
 3 | Redistribution and use in source and binary forms, with or without
 4 | modification, are permitted provided that the following conditions are
 5 | met:
 6 | 
 7 |    * Redistributions of source code must retain the above copyright
 8 | notice, this list of conditions and the following disclaimer.
 9 |    * Redistributions in binary form must reproduce the above
10 | copyright notice, this list of conditions and the following disclaimer
11 | in the documentation and/or other materials provided with the
12 | distribution.
13 |    * Neither the names of the authors nor the names of the
14 | contributors may be used to endorse or promote products derived from
15 | this software without specific prior written permission.
16 | 
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/LICENSE-MMAP-GO:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2011, Evan Shaw <edsrzf@gmail.com>
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without
 5 | modification, are permitted provided that the following conditions are met:
 6 |     * Redistributions of source code must retain the above copyright
 7 |       notice, this list of conditions and the following disclaimer.
 8 |     * Redistributions in binary form must reproduce the above copyright
 9 |       notice, this list of conditions and the following disclaimer in the
10 |       documentation and/or other materials provided with the distribution.
11 |     * Neither the name of the copyright holder nor the
12 |       names of its contributors may be used to endorse or promote products
13 |       derived from this software without specific prior written permission.
14 | 
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 | DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 | 
26 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2017 The Memory Authors. All rights reserved.
 2 | # Use of this source code is governed by a BSD-style
 3 | # license that can be found in the LICENSE file.
 4 | 
 5 | .PHONY:	all clean cover cpu editor internalError later mem nuke todo edit
 6 | 
 7 | grep=--include=*.go --include=*.l --include=*.y --include=*.yy
 8 | ngrep='TODOOK\|parser\.go\|scanner\.go\|.*_string\.go'
 9 | 
10 | all: editor
11 | 	go vet 2>&1 | grep -v $(ngrep) || true
12 | 	golint 2>&1 | grep -v $(ngrep) || true
13 | 	make todo
14 | 	unused . || true
15 | 	misspell *.go
16 | 	gosimple || true
17 | 	maligned || true
18 | 	unconvert -apply
19 | 
20 | clean:
21 | 	go clean
22 | 	rm -f *~ *.test *.out
23 | 
24 | cover:
25 | 	t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t
26 | 
27 | cpu: clean
28 | 	go test -run @ -bench . -cpuprofile cpu.out
29 | 	go tool pprof -lines *.test cpu.out
30 | 
31 | edit:
32 | 	@ 1>/dev/null 2>/dev/null gvim -p Makefile *.go
33 | 
34 | editor:
35 | 	gofmt -l -s -w *.go
36 | 	go test -i
37 | 	GOARCH=386 go build
38 | 	GOARCH=amd64 go build
39 | 	go test 2>&1 | tee log
40 | 
41 | internalError:
42 | 	egrep -ho '"internal error.*"' *.go | sort | cat -n
43 | 
44 | later:
45 | 	@grep -n $(grep) LATER * || true
46 | 	@grep -n $(grep) MAYBE * || true
47 | 
48 | mem: clean
49 | 	go test -run @ -bench . -memprofile mem.out -memprofilerate 1 -timeout 24h
50 | 	go tool pprof -lines -web -alloc_space *.test mem.out
51 | 
52 | nuke: clean
53 | 	go clean -i
54 | 
55 | todo:
56 | 	@grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * | grep -v $(ngrep) || true
57 | 	@grep -nr $(grep) TODO * | grep -v $(ngrep) || true
58 | 	@grep -nr $(grep) BUG * | grep -v $(ngrep) || true
59 | 	@grep -nr $(grep) [^[:alpha:]]println * | grep -v $(ngrep) || true
60 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | `github.com/cznic/memory` has moved to [`modernc.org/memory`](https://godoc.org/modernc.org/memory) ([vcs](https://gitlab.com/cznic/memory)).
2 | 
3 | Please update your import paths to `modernc.org/memory`.
4 | 
5 | This repo is now archived.
6 | 


--------------------------------------------------------------------------------
/all_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2017 The Memory Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | package memory
  6 | 
  7 | import (
  8 | 	"bytes"
  9 | 	"fmt"
 10 | 	"math"
 11 | 	"os"
 12 | 	"path"
 13 | 	"runtime"
 14 | 	"strings"
 15 | 	"testing"
 16 | 	"unsafe"
 17 | 
 18 | 	"github.com/cznic/mathutil"
 19 | )
 20 | 
 21 | func caller(s string, va ...interface{}) {
 22 | 	if s == "" {
 23 | 		s = strings.Repeat("%v ", len(va))
 24 | 	}
 25 | 	_, fn, fl, _ := runtime.Caller(2)
 26 | 	fmt.Fprintf(os.Stderr, "# caller: %s:%d: ", path.Base(fn), fl)
 27 | 	fmt.Fprintf(os.Stderr, s, va...)
 28 | 	fmt.Fprintln(os.Stderr)
 29 | 	_, fn, fl, _ = runtime.Caller(1)
 30 | 	fmt.Fprintf(os.Stderr, "# \tcallee: %s:%d: ", path.Base(fn), fl)
 31 | 	fmt.Fprintln(os.Stderr)
 32 | 	os.Stderr.Sync()
 33 | }
 34 | 
 35 | func dbg(s string, va ...interface{}) {
 36 | 	if s == "" {
 37 | 		s = strings.Repeat("%v ", len(va))
 38 | 	}
 39 | 	_, fn, fl, _ := runtime.Caller(1)
 40 | 	fmt.Fprintf(os.Stderr, "# dbg %s:%d: ", path.Base(fn), fl)
 41 | 	fmt.Fprintf(os.Stderr, s, va...)
 42 | 	fmt.Fprintln(os.Stderr)
 43 | 	os.Stderr.Sync()
 44 | }
 45 | 
 46 | func TODO(...interface{}) string { //TODOOK
 47 | 	_, fn, fl, _ := runtime.Caller(1)
 48 | 	return fmt.Sprintf("# TODO: %s:%d:\n", path.Base(fn), fl) //TODOOK
 49 | }
 50 | 
 51 | func use(...interface{}) {}
 52 | 
 53 | func init() {
 54 | 	use(caller, dbg, TODO) //TODOOK
 55 | }
 56 | 
 57 | // ============================================================================
 58 | 
 59 | const quota = 128 << 20
 60 | 
 61 | var (
 62 | 	max    = 2 * osPageSize
 63 | 	bigMax = 2 * pageSize
 64 | )
 65 | 
 66 | type block struct {
 67 | 	p    uintptr
 68 | 	size int
 69 | }
 70 | 
 71 | func test1u(t *testing.T, max int) {
 72 | 	var alloc Allocator
 73 | 	rem := quota
 74 | 	var a []block
 75 | 	srng, err := mathutil.NewFC32(0, math.MaxInt32, true)
 76 | 	if err != nil {
 77 | 		t.Fatal(err)
 78 | 	}
 79 | 
 80 | 	vrng, err := mathutil.NewFC32(0, math.MaxInt32, true)
 81 | 	if err != nil {
 82 | 		t.Fatal(err)
 83 | 	}
 84 | 
 85 | 	// Allocate
 86 | 	for rem > 0 {
 87 | 		size := srng.Next()%max + 1
 88 | 		rem -= size
 89 | 		p, err := alloc.UintptrMalloc(size)
 90 | 		if err != nil {
 91 | 			t.Fatal(err)
 92 | 		}
 93 | 
 94 | 		a = append(a, block{p, size})
 95 | 		for i := 0; i < size; i++ {
 96 | 			*(*byte)(unsafe.Pointer(p + uintptr(i))) = byte(vrng.Next())
 97 | 		}
 98 | 	}
 99 | 	t.Logf("allocs %v, mmaps %v, bytes %v, overhead %v (%.2f%%).", alloc.allocs, alloc.mmaps, alloc.bytes, alloc.bytes-quota, 100*float64(alloc.bytes-quota)/quota)
100 | 	srng.Seek(0)
101 | 	vrng.Seek(0)
102 | 	// Verify
103 | 	for i, b := range a {
104 | 		if g, e := b.size, srng.Next()%max+1; g != e {
105 | 			t.Fatal(i, g, e)
106 | 		}
107 | 
108 | 		if a, b := b.size, UintptrUsableSize(b.p); a > b {
109 | 			t.Fatal(i, a, b)
110 | 		}
111 | 
112 | 		for j := 0; j < b.size; j++ {
113 | 			g := *(*byte)(unsafe.Pointer(b.p + uintptr(j)))
114 | 			if e := byte(vrng.Next()); g != e {
115 | 				t.Fatalf("%v,%v %#x: %#02x %#02x", i, j, b.p+uintptr(j), g, e)
116 | 			}
117 | 
118 | 			*(*byte)(unsafe.Pointer(b.p + uintptr(j))) = 0
119 | 		}
120 | 	}
121 | 	// Shuffle
122 | 	for i := range a {
123 | 		j := srng.Next() % len(a)
124 | 		a[i], a[j] = a[j], a[i]
125 | 	}
126 | 	// Free
127 | 	for _, b := range a {
128 | 		if err := alloc.UintptrFree(b.p); err != nil {
129 | 			t.Fatal(err)
130 | 		}
131 | 	}
132 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
133 | 		t.Fatalf("%+v", alloc)
134 | 	}
135 | }
136 | 
137 | func Test1USmall(t *testing.T) { test1u(t, max) }
138 | func Test1UBig(t *testing.T)   { test1u(t, bigMax) }
139 | 
140 | func test2u(t *testing.T, max int) {
141 | 	var alloc Allocator
142 | 	rem := quota
143 | 	var a []block
144 | 	srng, err := mathutil.NewFC32(0, math.MaxInt32, true)
145 | 	if err != nil {
146 | 		t.Fatal(err)
147 | 	}
148 | 
149 | 	vrng, err := mathutil.NewFC32(0, math.MaxInt32, true)
150 | 	if err != nil {
151 | 		t.Fatal(err)
152 | 	}
153 | 
154 | 	// Allocate
155 | 	for rem > 0 {
156 | 		size := srng.Next()%max + 1
157 | 		rem -= size
158 | 		p, err := alloc.UintptrMalloc(size)
159 | 		if err != nil {
160 | 			t.Fatal(err)
161 | 		}
162 | 
163 | 		a = append(a, block{p, size})
164 | 		for i := 0; i < size; i++ {
165 | 			*(*byte)(unsafe.Pointer(p + uintptr(i))) = byte(vrng.Next())
166 | 		}
167 | 	}
168 | 	t.Logf("allocs %v, mmaps %v, bytes %v, overhead %v (%.2f%%).", alloc.allocs, alloc.mmaps, alloc.bytes, alloc.bytes-quota, 100*float64(alloc.bytes-quota)/quota)
169 | 	srng.Seek(0)
170 | 	vrng.Seek(0)
171 | 	// Verify & free
172 | 	for i, b := range a {
173 | 		if g, e := b.size, srng.Next()%max+1; g != e {
174 | 			t.Fatal(i, g, e)
175 | 		}
176 | 
177 | 		if a, b := b.size, UintptrUsableSize(b.p); a > b {
178 | 			t.Fatal(i, a, b)
179 | 		}
180 | 
181 | 		for j := 0; j < b.size; j++ {
182 | 			g := *(*byte)(unsafe.Pointer(b.p + uintptr(j)))
183 | 			if e := byte(vrng.Next()); g != e {
184 | 				t.Fatalf("%v,%v %#x: %#02x %#02x", i, j, b.p+uintptr(j), g, e)
185 | 			}
186 | 
187 | 			*(*byte)(unsafe.Pointer(b.p + uintptr(j))) = 0
188 | 		}
189 | 		if err := alloc.UintptrFree(b.p); err != nil {
190 | 			t.Fatal(err)
191 | 		}
192 | 	}
193 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
194 | 		t.Fatalf("%+v", alloc)
195 | 	}
196 | }
197 | 
198 | func Test2USmall(t *testing.T) { test2u(t, max) }
199 | func Test2UBig(t *testing.T)   { test2u(t, bigMax) }
200 | 
201 | func test3u(t *testing.T, max int) {
202 | 	var alloc Allocator
203 | 	rem := quota
204 | 	m := map[block][]byte{}
205 | 	srng, err := mathutil.NewFC32(1, max, true)
206 | 	if err != nil {
207 | 		t.Fatal(err)
208 | 	}
209 | 
210 | 	vrng, err := mathutil.NewFC32(1, max, true)
211 | 	if err != nil {
212 | 		t.Fatal(err)
213 | 	}
214 | 
215 | 	for rem > 0 {
216 | 		switch srng.Next() % 3 {
217 | 		case 0, 1: // 2/3 allocate
218 | 			size := srng.Next()
219 | 			rem -= size
220 | 			p, err := alloc.UintptrMalloc(size)
221 | 			if err != nil {
222 | 				t.Fatal(err)
223 | 			}
224 | 
225 | 			b := make([]byte, size)
226 | 			for i := range b {
227 | 				b[i] = byte(vrng.Next())
228 | 				*(*byte)(unsafe.Pointer(p + uintptr(i))) = b[i]
229 | 			}
230 | 			m[block{p, size}] = append([]byte(nil), b...)
231 | 		default: // 1/3 free
232 | 			for b, v := range m {
233 | 				for i, v := range v {
234 | 					if *(*byte)(unsafe.Pointer(b.p + uintptr(i))) != v {
235 | 						t.Fatal("corrupted heap")
236 | 					}
237 | 				}
238 | 
239 | 				if a, b := b.size, UintptrUsableSize(b.p); a > b {
240 | 					t.Fatal(a, b)
241 | 				}
242 | 
243 | 				for j := 0; j < b.size; j++ {
244 | 					*(*byte)(unsafe.Pointer(b.p + uintptr(j))) = 0
245 | 				}
246 | 				rem += b.size
247 | 				alloc.UintptrFree(b.p)
248 | 				delete(m, b)
249 | 				break
250 | 			}
251 | 		}
252 | 	}
253 | 	t.Logf("allocs %v, mmaps %v, bytes %v, overhead %v (%.2f%%).", alloc.allocs, alloc.mmaps, alloc.bytes, alloc.bytes-quota, 100*float64(alloc.bytes-quota)/quota)
254 | 	for b, v := range m {
255 | 		for i, v := range v {
256 | 			if *(*byte)(unsafe.Pointer(b.p + uintptr(i))) != v {
257 | 				t.Fatal("corrupted heap")
258 | 			}
259 | 		}
260 | 
261 | 		if a, b := b.size, UintptrUsableSize(b.p); a > b {
262 | 			t.Fatal(a, b)
263 | 		}
264 | 
265 | 		for j := 0; j < b.size; j++ {
266 | 			*(*byte)(unsafe.Pointer(b.p + uintptr(j))) = 0
267 | 		}
268 | 		alloc.UintptrFree(b.p)
269 | 	}
270 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
271 | 		t.Fatalf("%+v", alloc)
272 | 	}
273 | }
274 | 
275 | func Test3USmall(t *testing.T) { test3u(t, max) }
276 | func Test3UBig(t *testing.T)   { test3u(t, bigMax) }
277 | 
278 | func TestUFree(t *testing.T) {
279 | 	var alloc Allocator
280 | 	p, err := alloc.UintptrMalloc(1)
281 | 	if err != nil {
282 | 		t.Fatal(err)
283 | 	}
284 | 
285 | 	if err := alloc.UintptrFree(p); err != nil {
286 | 		t.Fatal(err)
287 | 	}
288 | 
289 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
290 | 		t.Fatalf("%+v", alloc)
291 | 	}
292 | }
293 | 
294 | func TestUMalloc(t *testing.T) {
295 | 	var alloc Allocator
296 | 	p, err := alloc.UintptrMalloc(maxSlotSize)
297 | 	if err != nil {
298 | 		t.Fatal(err)
299 | 	}
300 | 
301 | 	pg := (*page)(unsafe.Pointer(p &^ uintptr(osPageMask)))
302 | 	if 1<<pg.log > maxSlotSize {
303 | 		t.Fatal(1<<pg.log, maxSlotSize)
304 | 	}
305 | 
306 | 	if err := alloc.UintptrFree(p); err != nil {
307 | 		t.Fatal(err)
308 | 	}
309 | 
310 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
311 | 		t.Fatalf("%+v", alloc)
312 | 	}
313 | }
314 | 
315 | func test1(t *testing.T, max int) {
316 | 	var alloc Allocator
317 | 	rem := quota
318 | 	var a [][]byte
319 | 	srng, err := mathutil.NewFC32(0, math.MaxInt32, true)
320 | 	if err != nil {
321 | 		t.Fatal(err)
322 | 	}
323 | 
324 | 	vrng, err := mathutil.NewFC32(0, math.MaxInt32, true)
325 | 	if err != nil {
326 | 		t.Fatal(err)
327 | 	}
328 | 
329 | 	// Allocate
330 | 	for rem > 0 {
331 | 		size := srng.Next()%max + 1
332 | 		rem -= size
333 | 		b, err := alloc.Malloc(size)
334 | 		if err != nil {
335 | 			t.Fatal(err)
336 | 		}
337 | 
338 | 		a = append(a, b)
339 | 		for i := range b {
340 | 			b[i] = byte(vrng.Next())
341 | 		}
342 | 	}
343 | 	t.Logf("allocs %v, mmaps %v, bytes %v, overhead %v (%.2f%%).", alloc.allocs, alloc.mmaps, alloc.bytes, alloc.bytes-quota, 100*float64(alloc.bytes-quota)/quota)
344 | 	srng.Seek(0)
345 | 	vrng.Seek(0)
346 | 	// Verify
347 | 	for i, b := range a {
348 | 		if g, e := len(b), srng.Next()%max+1; g != e {
349 | 			t.Fatal(i, g, e)
350 | 		}
351 | 
352 | 		if a, b := len(b), UsableSize(&b[0]); a > b {
353 | 			t.Fatal(i, a, b)
354 | 		}
355 | 
356 | 		for i, g := range b {
357 | 			if e := byte(vrng.Next()); g != e {
358 | 				t.Fatalf("%v %p: %#02x %#02x", i, &b[i], g, e)
359 | 			}
360 | 
361 | 			b[i] = 0
362 | 		}
363 | 	}
364 | 	// Shuffle
365 | 	for i := range a {
366 | 		j := srng.Next() % len(a)
367 | 		a[i], a[j] = a[j], a[i]
368 | 	}
369 | 	// Free
370 | 	for _, b := range a {
371 | 		if err := alloc.Free(b); err != nil {
372 | 			t.Fatal(err)
373 | 		}
374 | 	}
375 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
376 | 		t.Fatalf("%+v", alloc)
377 | 	}
378 | }
379 | 
380 | func Test1Small(t *testing.T) { test1(t, max) }
381 | func Test1Big(t *testing.T)   { test1(t, bigMax) }
382 | 
383 | func test2(t *testing.T, max int) {
384 | 	var alloc Allocator
385 | 	rem := quota
386 | 	var a [][]byte
387 | 	srng, err := mathutil.NewFC32(0, math.MaxInt32, true)
388 | 	if err != nil {
389 | 		t.Fatal(err)
390 | 	}
391 | 
392 | 	vrng, err := mathutil.NewFC32(0, math.MaxInt32, true)
393 | 	if err != nil {
394 | 		t.Fatal(err)
395 | 	}
396 | 
397 | 	// Allocate
398 | 	for rem > 0 {
399 | 		size := srng.Next()%max + 1
400 | 		rem -= size
401 | 		b, err := alloc.Malloc(size)
402 | 		if err != nil {
403 | 			t.Fatal(err)
404 | 		}
405 | 
406 | 		a = append(a, b)
407 | 		for i := range b {
408 | 			b[i] = byte(vrng.Next())
409 | 		}
410 | 	}
411 | 	t.Logf("allocs %v, mmaps %v, bytes %v, overhead %v (%.2f%%).", alloc.allocs, alloc.mmaps, alloc.bytes, alloc.bytes-quota, 100*float64(alloc.bytes-quota)/quota)
412 | 	srng.Seek(0)
413 | 	vrng.Seek(0)
414 | 	// Verify & free
415 | 	for i, b := range a {
416 | 		if g, e := len(b), srng.Next()%max+1; g != e {
417 | 			t.Fatal(i, g, e)
418 | 		}
419 | 
420 | 		if a, b := len(b), UsableSize(&b[0]); a > b {
421 | 			t.Fatal(i, a, b)
422 | 		}
423 | 
424 | 		for i, g := range b {
425 | 			if e := byte(vrng.Next()); g != e {
426 | 				t.Fatalf("%v %p: %#02x %#02x", i, &b[i], g, e)
427 | 			}
428 | 
429 | 			b[i] = 0
430 | 		}
431 | 		if err := alloc.Free(b); err != nil {
432 | 			t.Fatal(err)
433 | 		}
434 | 	}
435 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
436 | 		t.Fatalf("%+v", alloc)
437 | 	}
438 | }
439 | 
440 | func Test2Small(t *testing.T) { test2(t, max) }
441 | func Test2Big(t *testing.T)   { test2(t, bigMax) }
442 | 
443 | func test3(t *testing.T, max int) {
444 | 	var alloc Allocator
445 | 	rem := quota
446 | 	m := map[*[]byte][]byte{}
447 | 	srng, err := mathutil.NewFC32(1, max, true)
448 | 	if err != nil {
449 | 		t.Fatal(err)
450 | 	}
451 | 
452 | 	vrng, err := mathutil.NewFC32(1, max, true)
453 | 	if err != nil {
454 | 		t.Fatal(err)
455 | 	}
456 | 
457 | 	for rem > 0 {
458 | 		switch srng.Next() % 3 {
459 | 		case 0, 1: // 2/3 allocate
460 | 			size := srng.Next()
461 | 			rem -= size
462 | 			b, err := alloc.Malloc(size)
463 | 			if err != nil {
464 | 				t.Fatal(err)
465 | 			}
466 | 
467 | 			for i := range b {
468 | 				b[i] = byte(vrng.Next())
469 | 			}
470 | 			m[&b] = append([]byte(nil), b...)
471 | 		default: // 1/3 free
472 | 			for k, v := range m {
473 | 				b := *k
474 | 				if !bytes.Equal(b, v) {
475 | 					t.Fatal("corrupted heap")
476 | 				}
477 | 
478 | 				if a, b := len(b), UsableSize(&b[0]); a > b {
479 | 					t.Fatal(a, b)
480 | 				}
481 | 
482 | 				for i := range b {
483 | 					b[i] = 0
484 | 				}
485 | 				rem += len(b)
486 | 				alloc.Free(b)
487 | 				delete(m, k)
488 | 				break
489 | 			}
490 | 		}
491 | 	}
492 | 	t.Logf("allocs %v, mmaps %v, bytes %v, overhead %v (%.2f%%).", alloc.allocs, alloc.mmaps, alloc.bytes, alloc.bytes-quota, 100*float64(alloc.bytes-quota)/quota)
493 | 	for k, v := range m {
494 | 		b := *k
495 | 		if !bytes.Equal(b, v) {
496 | 			t.Fatal("corrupted heap")
497 | 		}
498 | 
499 | 		if a, b := len(b), UsableSize(&b[0]); a > b {
500 | 			t.Fatal(a, b)
501 | 		}
502 | 
503 | 		for i := range b {
504 | 			b[i] = 0
505 | 		}
506 | 		alloc.Free(b)
507 | 	}
508 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
509 | 		t.Fatalf("%+v", alloc)
510 | 	}
511 | }
512 | 
513 | func Test3Small(t *testing.T) { test3(t, max) }
514 | func Test3Big(t *testing.T)   { test3(t, bigMax) }
515 | 
516 | func TestFree(t *testing.T) {
517 | 	var alloc Allocator
518 | 	b, err := alloc.Malloc(1)
519 | 	if err != nil {
520 | 		t.Fatal(err)
521 | 	}
522 | 
523 | 	if err := alloc.Free(b[:0]); err != nil {
524 | 		t.Fatal(err)
525 | 	}
526 | 
527 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
528 | 		t.Fatalf("%+v", alloc)
529 | 	}
530 | }
531 | 
532 | func TestMalloc(t *testing.T) {
533 | 	var alloc Allocator
534 | 	b, err := alloc.Malloc(maxSlotSize)
535 | 	if err != nil {
536 | 		t.Fatal(err)
537 | 	}
538 | 
539 | 	p := (*page)(unsafe.Pointer(uintptr(unsafe.Pointer(&b[0])) &^ uintptr(osPageMask)))
540 | 	if 1<<p.log > maxSlotSize {
541 | 		t.Fatal(1<<p.log, maxSlotSize)
542 | 	}
543 | 
544 | 	if err := alloc.Free(b[:0]); err != nil {
545 | 		t.Fatal(err)
546 | 	}
547 | 
548 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
549 | 		t.Fatalf("%+v", alloc)
550 | 	}
551 | }
552 | 
553 | func benchmarkFree(b *testing.B, size int) {
554 | 	var alloc Allocator
555 | 	a := make([][]byte, b.N)
556 | 	for i := range a {
557 | 		p, err := alloc.Malloc(size)
558 | 		if err != nil {
559 | 			b.Fatal(err)
560 | 		}
561 | 
562 | 		a[i] = p
563 | 	}
564 | 	b.ResetTimer()
565 | 	for _, b := range a {
566 | 		alloc.Free(b)
567 | 	}
568 | 	b.StopTimer()
569 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
570 | 		b.Fatalf("%+v", alloc)
571 | 	}
572 | }
573 | 
574 | func BenchmarkFree16(b *testing.B) { benchmarkFree(b, 1<<4) }
575 | func BenchmarkFree32(b *testing.B) { benchmarkFree(b, 1<<5) }
576 | func BenchmarkFree64(b *testing.B) { benchmarkFree(b, 1<<6) }
577 | 
578 | func benchmarkCalloc(b *testing.B, size int) {
579 | 	var alloc Allocator
580 | 	a := make([][]byte, b.N)
581 | 	b.ResetTimer()
582 | 	for i := range a {
583 | 		p, err := alloc.Calloc(size)
584 | 		if err != nil {
585 | 			b.Fatal(err)
586 | 		}
587 | 
588 | 		a[i] = p
589 | 	}
590 | 	b.StopTimer()
591 | 	for _, b := range a {
592 | 		alloc.Free(b)
593 | 	}
594 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
595 | 		b.Fatalf("%+v", alloc)
596 | 	}
597 | }
598 | 
599 | func BenchmarkCalloc16(b *testing.B) { benchmarkCalloc(b, 1<<4) }
600 | func BenchmarkCalloc32(b *testing.B) { benchmarkCalloc(b, 1<<5) }
601 | func BenchmarkCalloc64(b *testing.B) { benchmarkCalloc(b, 1<<6) }
602 | 
603 | func benchmarkGoCalloc(b *testing.B, size int) {
604 | 	a := make([][]byte, b.N)
605 | 	b.ResetTimer()
606 | 	for i := range a {
607 | 		a[i] = make([]byte, size)
608 | 	}
609 | 	b.StopTimer()
610 | 	use(a)
611 | }
612 | 
613 | func BenchmarkGoCalloc16(b *testing.B) { benchmarkGoCalloc(b, 1<<4) }
614 | func BenchmarkGoCalloc32(b *testing.B) { benchmarkGoCalloc(b, 1<<5) }
615 | func BenchmarkGoCalloc64(b *testing.B) { benchmarkGoCalloc(b, 1<<6) }
616 | 
617 | func benchmarkMalloc(b *testing.B, size int) {
618 | 	var alloc Allocator
619 | 	a := make([][]byte, b.N)
620 | 	b.ResetTimer()
621 | 	for i := range a {
622 | 		p, err := alloc.Malloc(size)
623 | 		if err != nil {
624 | 			b.Fatal(err)
625 | 		}
626 | 
627 | 		a[i] = p
628 | 	}
629 | 	b.StopTimer()
630 | 	for _, b := range a {
631 | 		alloc.Free(b)
632 | 	}
633 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
634 | 		b.Fatalf("%+v", alloc)
635 | 	}
636 | }
637 | 
638 | func BenchmarkMalloc16(b *testing.B) { benchmarkMalloc(b, 1<<4) }
639 | func BenchmarkMalloc32(b *testing.B) { benchmarkMalloc(b, 1<<5) }
640 | func BenchmarkMalloc64(b *testing.B) { benchmarkMalloc(b, 1<<6) }
641 | 
642 | func benchmarkUintptrFree(b *testing.B, size int) {
643 | 	var alloc Allocator
644 | 	a := make([]uintptr, b.N)
645 | 	for i := range a {
646 | 		p, err := alloc.UintptrMalloc(size)
647 | 		if err != nil {
648 | 			b.Fatal(err)
649 | 		}
650 | 
651 | 		a[i] = p
652 | 	}
653 | 	b.ResetTimer()
654 | 	for _, p := range a {
655 | 		alloc.UintptrFree(p)
656 | 	}
657 | 	b.StopTimer()
658 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
659 | 		b.Fatalf("%+v", alloc)
660 | 	}
661 | }
662 | 
663 | func BenchmarkUintptrFree16(b *testing.B) { benchmarkUintptrFree(b, 1<<4) }
664 | func BenchmarkUintptrFree32(b *testing.B) { benchmarkUintptrFree(b, 1<<5) }
665 | func BenchmarkUintptrFree64(b *testing.B) { benchmarkUintptrFree(b, 1<<6) }
666 | 
667 | func benchmarkUintptrCalloc(b *testing.B, size int) {
668 | 	var alloc Allocator
669 | 	a := make([]uintptr, b.N)
670 | 	b.ResetTimer()
671 | 	for i := range a {
672 | 		p, err := alloc.UintptrCalloc(size)
673 | 		if err != nil {
674 | 			b.Fatal(err)
675 | 		}
676 | 
677 | 		a[i] = p
678 | 	}
679 | 	b.StopTimer()
680 | 	for _, p := range a {
681 | 		alloc.UintptrFree(p)
682 | 	}
683 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
684 | 		b.Fatalf("%+v", alloc)
685 | 	}
686 | }
687 | 
688 | func BenchmarkUintptrCalloc16(b *testing.B) { benchmarkUintptrCalloc(b, 1<<4) }
689 | func BenchmarkUintptrCalloc32(b *testing.B) { benchmarkUintptrCalloc(b, 1<<5) }
690 | func BenchmarkUintptrCalloc64(b *testing.B) { benchmarkUintptrCalloc(b, 1<<6) }
691 | 
692 | func benchmarkUintptrMalloc(b *testing.B, size int) {
693 | 	var alloc Allocator
694 | 	a := make([]uintptr, b.N)
695 | 	b.ResetTimer()
696 | 	for i := range a {
697 | 		p, err := alloc.UintptrMalloc(size)
698 | 		if err != nil {
699 | 			b.Fatal(err)
700 | 		}
701 | 
702 | 		a[i] = p
703 | 	}
704 | 	b.StopTimer()
705 | 	for _, p := range a {
706 | 		alloc.UintptrFree(p)
707 | 	}
708 | 	if alloc.allocs != 0 || alloc.mmaps != 0 || alloc.bytes != 0 || len(alloc.regs) != 0 {
709 | 		b.Fatalf("%+v", alloc)
710 | 	}
711 | }
712 | 
713 | func BenchmarkUintptrMalloc16(b *testing.B) { benchmarkUintptrMalloc(b, 1<<4) }
714 | func BenchmarkUintptrMalloc32(b *testing.B) { benchmarkUintptrMalloc(b, 1<<5) }
715 | func BenchmarkUintptrMalloc64(b *testing.B) { benchmarkUintptrMalloc(b, 1<<6) }
716 | 


--------------------------------------------------------------------------------
/memory.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2017 The Memory Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // Package memory implements a memory allocator.
  6 | //
  7 | // Changelog
  8 | //
  9 | // 2017-10-03 Added alternative, unsafe.Pointer-based API.
 10 | //
 11 | // Benchmarks
 12 | //
 13 | // Intel® Core™ i5-4670 CPU @ 3.40GHz × 4
 14 | //
 15 | //  goos: linux
 16 | //  goarch: amd64
 17 | //  pkg: github.com/cznic/memory
 18 | //  BenchmarkFree16-4           	100000000	        15.3 ns/op	       0 B/op	       0 allocs/op
 19 | //  BenchmarkFree32-4           	100000000	        21.3 ns/op	       0 B/op	       0 allocs/op
 20 | //  BenchmarkFree64-4           	50000000	        35.9 ns/op	       0 B/op	       0 allocs/op
 21 | //  BenchmarkCalloc16-4         	50000000	        26.6 ns/op	       0 B/op	       0 allocs/op
 22 | //  BenchmarkCalloc32-4         	50000000	        30.1 ns/op	       0 B/op	       0 allocs/op
 23 | //  BenchmarkCalloc64-4         	30000000	        38.1 ns/op	       0 B/op	       0 allocs/op
 24 | //  BenchmarkGoCalloc16-4       	50000000	        29.3 ns/op	      16 B/op	       1 allocs/op
 25 | //  BenchmarkGoCalloc32-4       	50000000	        30.4 ns/op	      32 B/op	       1 allocs/op
 26 | //  BenchmarkGoCalloc64-4       	30000000	        37.9 ns/op	      64 B/op	       1 allocs/op
 27 | //  BenchmarkMalloc16-4         	100000000	        15.4 ns/op	       0 B/op	       0 allocs/op
 28 | //  BenchmarkMalloc32-4         	100000000	        15.6 ns/op	       0 B/op	       0 allocs/op
 29 | //  BenchmarkMalloc64-4         	100000000	        15.9 ns/op	       0 B/op	       0 allocs/op
 30 | //  BenchmarkUnsafeFree16-4     	100000000	        14.4 ns/op	       0 B/op	       0 allocs/op
 31 | //  BenchmarkUnsafeFree32-4     	100000000	        20.4 ns/op	       0 B/op	       0 allocs/op
 32 | //  BenchmarkUnsafeFree64-4     	50000000	        34.1 ns/op	       0 B/op	       0 allocs/op
 33 | //  BenchmarkUnsafeCalloc16-4   	50000000	        23.2 ns/op	       0 B/op	       0 allocs/op
 34 | //  BenchmarkUnsafeCalloc32-4   	50000000	        28.0 ns/op	       0 B/op	       0 allocs/op
 35 | //  BenchmarkUnsafeCalloc64-4   	50000000	        34.1 ns/op	       0 B/op	       0 allocs/op
 36 | //  BenchmarkUnsafeMalloc16-4   	100000000	        13.8 ns/op	       0 B/op	       0 allocs/op
 37 | //  BenchmarkUnsafeMalloc32-4   	100000000	        14.2 ns/op	       0 B/op	       0 allocs/op
 38 | //  BenchmarkUnsafeMalloc64-4   	100000000	        14.0 ns/op	       0 B/op	       0 allocs/op
 39 | //  PASS
 40 | //  ok  	github.com/cznic/memory	229.054s
 41 | package memory
 42 | 
 43 | import (
 44 | 	"fmt"
 45 | 	"os"
 46 | 	"reflect"
 47 | 	"unsafe"
 48 | 
 49 | 	"github.com/cznic/mathutil"
 50 | )
 51 | 
 52 | const mallocAllign = 16 // Must be >= 16
 53 | 
 54 | var (
 55 | 	headerSize  = roundup(int(unsafe.Sizeof(page{})), mallocAllign)
 56 | 	maxSlotSize = pageAvail >> 1
 57 | 	osPageMask  = osPageSize - 1
 58 | 	osPageSize  = os.Getpagesize()
 59 | 	pageAvail   = pageSize - headerSize
 60 | 	pageMask    = pageSize - 1
 61 | )
 62 | 
 63 | // if n%m != 0 { n += m-n%m }. m must be a power of 2.
 64 | func roundup(n, m int) int { return (n + m - 1) &^ (m - 1) }
 65 | 
 66 | type node struct {
 67 | 	prev, next *node
 68 | }
 69 | 
 70 | type page struct {
 71 | 	brk  int
 72 | 	log  uint
 73 | 	size int
 74 | 	used int
 75 | }
 76 | 
 77 | // Allocator allocates and frees memory. Its zero value is ready for use.
 78 | type Allocator struct {
 79 | 	allocs int // # of allocs.
 80 | 	bytes  int // Asked from OS.
 81 | 	cap    [64]int
 82 | 	lists  [64]*node
 83 | 	mmaps  int // Asked from OS.
 84 | 	pages  [64]*page
 85 | 	regs   map[*page]struct{}
 86 | }
 87 | 
 88 | func (a *Allocator) mmap(size int) (*page, error) {
 89 | 	p, size, err := mmap(size)
 90 | 	if err != nil {
 91 | 		return nil, err
 92 | 	}
 93 | 
 94 | 	a.mmaps++
 95 | 	a.bytes += size
 96 | 	pg := (*page)(unsafe.Pointer(p))
 97 | 	if a.regs == nil {
 98 | 		a.regs = map[*page]struct{}{}
 99 | 	}
100 | 	pg.size = size
101 | 	a.regs[pg] = struct{}{}
102 | 	return pg, nil
103 | }
104 | 
105 | func (a *Allocator) newPage(size int) (*page, error) {
106 | 	size += headerSize
107 | 	p, err := a.mmap(size)
108 | 	if err != nil {
109 | 		return nil, err
110 | 	}
111 | 
112 | 	p.log = 0
113 | 	return p, nil
114 | }
115 | 
116 | func (a *Allocator) newSharedPage(log uint) (*page, error) {
117 | 	if a.cap[log] == 0 {
118 | 		a.cap[log] = pageAvail / (1 << log)
119 | 	}
120 | 	size := headerSize + a.cap[log]<<log
121 | 	p, err := a.mmap(size)
122 | 	if err != nil {
123 | 		return nil, err
124 | 	}
125 | 
126 | 	a.pages[log] = p
127 | 	p.log = log
128 | 	return p, nil
129 | }
130 | 
131 | func (a *Allocator) unmap(p *page) error {
132 | 	delete(a.regs, p)
133 | 	a.mmaps--
134 | 	return unmap(uintptr(unsafe.Pointer(p)), p.size)
135 | }
136 | 
137 | // UintptrCalloc is like Calloc except it returns an uintptr.
138 | func (a *Allocator) UintptrCalloc(size int) (r uintptr, err error) {
139 | 	if trace {
140 | 		defer func() {
141 | 			fmt.Fprintf(os.Stderr, "Calloc(%#x) %#x, %v\n", size, r, err)
142 | 		}()
143 | 	}
144 | 	if r, err = a.UintptrMalloc(size); r == 0 || err != nil {
145 | 		return 0, err
146 | 	}
147 | 	b := ((*rawmem)(unsafe.Pointer(r)))[:size]
148 | 	for i := range b {
149 | 		b[i] = 0
150 | 	}
151 | 	return r, nil
152 | }
153 | 
154 | // UintptrFree is like Free except its argument is an uintptr, which must have
155 | // been acquired from UintptrCalloc or UintptrMalloc or UintptrRealloc.
156 | func (a *Allocator) UintptrFree(p uintptr) (err error) {
157 | 	if trace {
158 | 		defer func() {
159 | 			fmt.Fprintf(os.Stderr, "Free(%#x) %v\n", p, err)
160 | 		}()
161 | 	}
162 | 	if p == 0 {
163 | 		return nil
164 | 	}
165 | 
166 | 	a.allocs--
167 | 	pg := (*page)(unsafe.Pointer(p &^ uintptr(pageMask)))
168 | 	log := pg.log
169 | 	if log == 0 {
170 | 		a.bytes -= pg.size
171 | 		return a.unmap(pg)
172 | 	}
173 | 
174 | 	n := (*node)(unsafe.Pointer(p))
175 | 	n.prev = nil
176 | 	n.next = a.lists[log]
177 | 	if n.next != nil {
178 | 		n.next.prev = n
179 | 	}
180 | 	a.lists[log] = n
181 | 	pg.used--
182 | 	if pg.used != 0 {
183 | 		return nil
184 | 	}
185 | 
186 | 	for i := 0; i < pg.brk; i++ {
187 | 		n := (*node)(unsafe.Pointer(uintptr(unsafe.Pointer(pg)) + uintptr(headerSize+i<<log)))
188 | 		switch {
189 | 		case n.prev == nil:
190 | 			a.lists[log] = n.next
191 | 			if n.next != nil {
192 | 				n.next.prev = nil
193 | 			}
194 | 		case n.next == nil:
195 | 			n.prev.next = nil
196 | 		default:
197 | 			n.prev.next = n.next
198 | 			n.next.prev = n.prev
199 | 		}
200 | 	}
201 | 
202 | 	if a.pages[log] == pg {
203 | 		a.pages[log] = nil
204 | 	}
205 | 	a.bytes -= pg.size
206 | 	return a.unmap(pg)
207 | }
208 | 
209 | // UintptrMalloc is like Malloc except it returns an uinptr.
210 | func (a *Allocator) UintptrMalloc(size int) (r uintptr, err error) {
211 | 	if trace {
212 | 		defer func() {
213 | 			fmt.Fprintf(os.Stderr, "Malloc(%#x) %#x, %v\n", size, r, err)
214 | 		}()
215 | 	}
216 | 	if size < 0 {
217 | 		panic("invalid malloc size")
218 | 	}
219 | 
220 | 	if size == 0 {
221 | 		return 0, nil
222 | 	}
223 | 
224 | 	a.allocs++
225 | 	log := uint(mathutil.BitLen(roundup(size, mallocAllign) - 1))
226 | 	if 1<<log > maxSlotSize {
227 | 		p, err := a.newPage(size)
228 | 		if err != nil {
229 | 			return 0, err
230 | 		}
231 | 
232 | 		return uintptr(unsafe.Pointer(p)) + uintptr(headerSize), nil
233 | 	}
234 | 
235 | 	if a.lists[log] == nil && a.pages[log] == nil {
236 | 		if _, err := a.newSharedPage(log); err != nil {
237 | 			return 0, err
238 | 		}
239 | 	}
240 | 
241 | 	if p := a.pages[log]; p != nil {
242 | 		p.used++
243 | 		p.brk++
244 | 		if p.brk == a.cap[log] {
245 | 			a.pages[log] = nil
246 | 		}
247 | 		return uintptr(unsafe.Pointer(p)) + uintptr(headerSize+(p.brk-1)<<log), nil
248 | 	}
249 | 
250 | 	n := a.lists[log]
251 | 	p := (*page)(unsafe.Pointer(uintptr(unsafe.Pointer(n)) &^ uintptr(pageMask)))
252 | 	a.lists[log] = n.next
253 | 	if n.next != nil {
254 | 		n.next.prev = nil
255 | 	}
256 | 	p.used++
257 | 	return uintptr(unsafe.Pointer(n)), nil
258 | }
259 | 
260 | // UintptrRealloc is like Realloc except its first argument is an uintptr,
261 | // which must have been returned from UintptrCalloc, UintptrMalloc or
262 | // UintptrRealloc.
263 | func (a *Allocator) UintptrRealloc(p uintptr, size int) (r uintptr, err error) {
264 | 	if trace {
265 | 		defer func() {
266 | 			fmt.Fprintf(os.Stderr, "UnsafeRealloc(%#x, %#x) %#x, %v\n", p, size, r, err)
267 | 		}()
268 | 	}
269 | 	switch {
270 | 	case p == 0:
271 | 		return a.UintptrMalloc(size)
272 | 	case size == 0 && p != 0:
273 | 		return 0, a.UintptrFree(p)
274 | 	}
275 | 
276 | 	us := UintptrUsableSize(p)
277 | 	if us > size {
278 | 		return p, nil
279 | 	}
280 | 
281 | 	if r, err = a.UintptrMalloc(size); err != nil {
282 | 		return 0, err
283 | 	}
284 | 
285 | 	if us < size {
286 | 		size = us
287 | 	}
288 | 	copy((*rawmem)(unsafe.Pointer(r))[:size], (*rawmem)(unsafe.Pointer(p))[:size])
289 | 	return r, a.UintptrFree(p)
290 | }
291 | 
292 | // UintptrUsableSize is like UsableSize except its argument is an uintptr,
293 | // which must have been returned from UintptrCalloc, UintptrMalloc or
294 | // UintptrRealloc.
295 | func UintptrUsableSize(p uintptr) (r int) {
296 | 	if trace {
297 | 		defer func() {
298 | 			fmt.Fprintf(os.Stderr, "UsableSize(%#x) %#x\n", p, r)
299 | 		}()
300 | 	}
301 | 	if p == 0 {
302 | 		return 0
303 | 	}
304 | 
305 | 	return usableSize(p)
306 | }
307 | 
308 | func usableSize(p uintptr) (r int) {
309 | 	pg := (*page)(unsafe.Pointer(p &^ uintptr(pageMask)))
310 | 	if pg.log != 0 {
311 | 		return 1 << pg.log
312 | 	}
313 | 
314 | 	return pg.size - headerSize
315 | }
316 | 
317 | // Calloc is like Malloc except the allocated memory is zeroed.
318 | func (a *Allocator) Calloc(size int) (r []byte, err error) {
319 | 	p, err := a.UintptrCalloc(size)
320 | 	if err != nil {
321 | 		return nil, err
322 | 	}
323 | 
324 | 	var b []byte
325 | 	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
326 | 	sh.Cap = usableSize(p)
327 | 	sh.Data = p
328 | 	sh.Len = size
329 | 	return b, nil
330 | }
331 | 
332 | // Close releases all OS resources used by a and sets it to its zero value.
333 | //
334 | // It's not necessary to Close the Allocator when exiting a process.
335 | func (a *Allocator) Close() (err error) {
336 | 	for p := range a.regs {
337 | 		if e := a.unmap(p); e != nil && err == nil {
338 | 			err = e
339 | 		}
340 | 	}
341 | 	*a = Allocator{}
342 | 	return err
343 | }
344 | 
345 | // Free deallocates memory (as in C.free). The argument of Free must have been
346 | // acquired from Calloc or Malloc or Realloc.
347 | func (a *Allocator) Free(b []byte) (err error) {
348 | 	if b = b[:cap(b)]; len(b) == 0 {
349 | 		return nil
350 | 	}
351 | 
352 | 	return a.UintptrFree(uintptr(unsafe.Pointer(&b[0])))
353 | }
354 | 
355 | // Malloc allocates size bytes and returns a byte slice of the allocated
356 | // memory. The memory is not initialized. Malloc panics for size < 0 and
357 | // returns (nil, nil) for zero size.
358 | //
359 | // It's ok to reslice the returned slice but the result of appending to it
360 | // cannot be passed to Free or Realloc as it may refer to a different backing
361 | // array afterwards.
362 | func (a *Allocator) Malloc(size int) (r []byte, err error) {
363 | 	p, err := a.UintptrMalloc(size)
364 | 	if p == 0 || err != nil {
365 | 		return nil, err
366 | 	}
367 | 
368 | 	sh := (*reflect.SliceHeader)(unsafe.Pointer(&r))
369 | 	sh.Cap = usableSize(p)
370 | 	sh.Data = p
371 | 	sh.Len = size
372 | 	return r, nil
373 | }
374 | 
375 | // Realloc changes the size of the backing array of b to size bytes or returns
376 | // an error, if any.  The contents will be unchanged in the range from the
377 | // start of the region up to the minimum of the old and new  sizes.   If the
378 | // new size is larger than the old size, the added memory will not be
379 | // initialized.  If b's backing array is of zero size, then the call is
380 | // equivalent to Malloc(size), for all values of size; if size is equal to
381 | // zero, and b's backing array is not of zero size, then the call is equivalent
382 | // to Free(b).  Unless b's backing array is of zero size, it must have been
383 | // returned by an earlier call to Malloc, Calloc or Realloc.  If the area
384 | // pointed to was moved, a Free(b) is done.
385 | func (a *Allocator) Realloc(b []byte, size int) (r []byte, err error) {
386 | 	var p uintptr
387 | 	if b = b[:cap(b)]; len(b) != 0 {
388 | 		p = uintptr(unsafe.Pointer(&b[0]))
389 | 	}
390 | 	if p, err = a.UintptrRealloc(p, size); p == 0 || err != nil {
391 | 		return nil, err
392 | 	}
393 | 
394 | 	sh := (*reflect.SliceHeader)(unsafe.Pointer(&r))
395 | 	sh.Cap = usableSize(p)
396 | 	sh.Data = p
397 | 	sh.Len = size
398 | 	return r, nil
399 | }
400 | 
401 | // UsableSize reports the size of the memory block allocated at p, which must
402 | // point to the first byte of a slice returned from Calloc, Malloc or Realloc.
403 | // The allocated memory block size can be larger than the size originally
404 | // requested from Calloc, Malloc or Realloc.
405 | func UsableSize(p *byte) (r int) { return UintptrUsableSize(uintptr(unsafe.Pointer(p))) }
406 | 
407 | // UnsafeCalloc is like Calloc except it returns an unsafe.Pointer.
408 | func (a *Allocator) UnsafeCalloc(size int) (r unsafe.Pointer, err error) {
409 | 	p, err := a.UintptrCalloc(size)
410 | 	if err != nil {
411 | 		return nil, err
412 | 	}
413 | 
414 | 	return unsafe.Pointer(p), nil
415 | }
416 | 
417 | // UnsafeFree is like Free except its argument is an unsafe.Pointer, which must
418 | // have been acquired from UnsafeCalloc or UnsafeMalloc or UnsafeRealloc.
419 | func (a *Allocator) UnsafeFree(p unsafe.Pointer) (err error) { return a.UintptrFree(uintptr(p)) }
420 | 
421 | // UnsafeMalloc is like Malloc except it returns an unsafe.Pointer.
422 | func (a *Allocator) UnsafeMalloc(size int) (r unsafe.Pointer, err error) {
423 | 	p, err := a.UintptrMalloc(size)
424 | 	if err != nil {
425 | 		return nil, err
426 | 	}
427 | 
428 | 	return unsafe.Pointer(p), nil
429 | }
430 | 
431 | // UnsafeRealloc is like Realloc except its first argument is an
432 | // unsafe.Pointer, which must have been returned from UnsafeCalloc,
433 | // UnsafeMalloc or UnsafeRealloc.
434 | func (a *Allocator) UnsafeRealloc(p unsafe.Pointer, size int) (r unsafe.Pointer, err error) {
435 | 	q, err := a.UintptrRealloc(uintptr(p), size)
436 | 	if err != nil {
437 | 		return nil, err
438 | 	}
439 | 
440 | 	return unsafe.Pointer(q), nil
441 | }
442 | 
443 | // UnsafeUsableSize is like UsableSize except its argument is an
444 | // unsafe.Pointer, which must have been returned from UnsafeCalloc,
445 | // UnsafeMalloc or UnsafeRealloc.
446 | func UnsafeUsableSize(p unsafe.Pointer) (r int) { return UintptrUsableSize(uintptr(p)) }
447 | 


--------------------------------------------------------------------------------
/memory32.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2018 The Memory Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build 386 arm armbe mips mipsle ppc ppc64le s390 s390x sparc
 6 | 
 7 | package memory
 8 | 
 9 | type rawmem [1<<31 - 1]byte
10 | 


--------------------------------------------------------------------------------
/memory64.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2018 The Memory Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build amd64 amd64p32 arm64 arm64be mips64 mips64le mips64p32 mips64p32le ppc64 sparc64
 6 | 
 7 | package memory
 8 | 
 9 | type rawmem [1<<50 - 1]byte
10 | 


--------------------------------------------------------------------------------
/mmap_unix.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Evan Shaw. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE-MMAP-GO file.
 4 | 
 5 | // +build darwin dragonfly freebsd linux openbsd solaris netbsd
 6 | 
 7 | // Modifications (c) 2017 The Memory Authors.
 8 | 
 9 | package memory
10 | 
11 | import (
12 | 	"syscall"
13 | 	"unsafe"
14 | )
15 | 
16 | var pageSize = 1 << 20
17 | 
18 | func unmap(addr uintptr, size int) error {
19 | 	_, _, errno := syscall.Syscall(syscall.SYS_MUNMAP, addr, uintptr(size), 0)
20 | 	if errno != 0 {
21 | 		return errno
22 | 	}
23 | 
24 | 	return nil
25 | }
26 | 
27 | // pageSize aligned.
28 | func mmap(size int) (uintptr, int, error) {
29 | 	size = roundup(size, osPageSize)
30 | 	b, err := syscall.Mmap(-1, 0, size+pageSize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED|syscall.MAP_ANON)
31 | 	if err != nil {
32 | 		return 0, 0, err
33 | 	}
34 | 
35 | 	n := len(b)
36 | 	p := uintptr(unsafe.Pointer(&b[0]))
37 | 	if p&uintptr(osPageMask) != 0 {
38 | 		panic("internal error")
39 | 	}
40 | 
41 | 	mod := int(p) & pageMask
42 | 	if mod != 0 {
43 | 		m := pageSize - mod
44 | 		if err := unmap(p, m); err != nil {
45 | 			return 0, 0, err
46 | 		}
47 | 
48 | 		b = b[m:]
49 | 		n -= m
50 | 		p += uintptr(m)
51 | 	}
52 | 
53 | 	if p&uintptr(pageMask) != 0 {
54 | 		panic("internal error")
55 | 	}
56 | 
57 | 	if n-size != 0 {
58 | 		if err := unmap(p+uintptr(size), n-size); err != nil {
59 | 			return 0, 0, err
60 | 		}
61 | 	}
62 | 
63 | 	return p, size, nil
64 | }
65 | 


--------------------------------------------------------------------------------
/mmap_windows.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The Memory Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package memory
 6 | 
 7 | import (
 8 | 	"syscall"
 9 | )
10 | 
11 | const (
12 | 	_MEM_COMMIT   = 0x1000
13 | 	_MEM_RESERVE  = 0x2000
14 | 	_MEM_DECOMMIT = 0x4000
15 | 	_MEM_RELEASE  = 0x8000
16 | 
17 | 	_PAGE_READWRITE = 0x0004
18 | 	_PAGE_NOACCESS  = 0x0001
19 | )
20 | 
21 | var (
22 | 	pageSize = 1 << 16
23 | 
24 | 	modkernel32      = syscall.NewLazyDLL("kernel32.dll")
25 | 	procVirtualAlloc = modkernel32.NewProc("VirtualAlloc")
26 | 	procVirtualFree  = modkernel32.NewProc("VirtualFree")
27 | )
28 | 
29 | // pageSize aligned.
30 | func mmap(size int) (uintptr, int, error) {
31 | 	size = roundup(size, pageSize)
32 | 	addr, _, err := procVirtualAlloc.Call(0, uintptr(size), _MEM_COMMIT|_MEM_RESERVE, _PAGE_READWRITE)
33 | 	if err.(syscall.Errno) != 0 || addr == 0 {
34 | 		return addr, size, err
35 | 	}
36 | 	return addr, size, nil
37 | }
38 | 
39 | func unmap(addr uintptr, size int) error {
40 | 	r, _, err := procVirtualFree.Call(addr, 0, _MEM_RELEASE)
41 | 	if r == 0 {
42 | 		return err
43 | 	}
44 | 
45 | 	return nil
46 | }
47 | 


--------------------------------------------------------------------------------
/trace_disabled.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The Memory Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build !memory.trace
 6 | 
 7 | package memory
 8 | 
 9 | const trace = false
10 | 


--------------------------------------------------------------------------------
/trace_enabled.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The Memory Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build memory.trace
 6 | 
 7 | package memory
 8 | 
 9 | const trace = true
10 | 


--------------------------------------------------------------------------------