├── AUTHORS
├── CONTRIBUTORS
├── LICENSE
├── Makefile
├── README
├── README.md
├── all_test.go
├── falloc
    ├── LICENSE
    ├── README
    ├── all_test.go
    ├── docs.go
    ├── error.go
    ├── falloc.go
    └── test_deps.go
├── fileutil.go
├── fileutil_arm.go
├── fileutil_darwin.go
├── fileutil_dragonfly.go
├── fileutil_freebsd.go
├── fileutil_linux.go
├── fileutil_netbsd.go
├── fileutil_openbsd.go
├── fileutil_plan9.go
├── fileutil_solaris.go
├── fileutil_windows.go
├── hdb
    ├── LICENSE
    ├── README
    ├── all_test.go
    ├── hdb.go
    └── test_deps.go
├── punch_test.go
├── storage
    ├── LICENSE
    ├── README
    ├── all_test.go
    ├── cache.go
    ├── cache_test.go
    ├── dev_test.go
    ├── file.go
    ├── mem.go
    ├── mem_test.go
    ├── probe.go
    ├── probe_test.go
    ├── storage.go
    └── test_deps.go
└── test_deps.go


/AUTHORS:
--------------------------------------------------------------------------------
 1 | # This file lists authors for copyright purposes.  This file is distinct from
 2 | # the CONTRIBUTORS files.  See the latter for an explanation.
 3 | #
 4 | # Names should be added to this file as:
 5 | #     Name or Organization <email address>
 6 | #
 7 | # The email address is not required for organizations.
 8 | #
 9 | # Please keep the list sorted.
10 | 
11 | CZ.NIC z.s.p.o. <kontakt@nic.cz>
12 | Jan Mercl <0xjnml@gmail.com>
13 | Linelane GmbH <info@linelane.com>
14 | Aaron Bieber <deftly@gmail.com>
15 | Pierre-Alain TORET <pierre-alain.toreŧ@protonmail.com>
16 | 


--------------------------------------------------------------------------------
/CONTRIBUTORS:
--------------------------------------------------------------------------------
 1 | # This file lists people who contributed code to this repository.  The AUTHORS
 2 | # file lists the copyright holders; this file lists people.
 3 | #
 4 | # Names should be added to this file like so:
 5 | #     Name <email address>
 6 | #
 7 | # Please keep the list sorted.
 8 | 
 9 | Andris Valums <info@linelane.com>
10 | Bill Thiede <xinu.tv>
11 | Gary Burd <gary@beagledreams.com>
12 | Jan Mercl <0xjnml@gmail.com>
13 | Nick Owens <mischief@offblast.org>
14 | Tamás Gulácsi <gt-dev@gthomas.eu>
15 | Aaron Bieber <deftly@gmail.com>
16 | Pierre-Alain TORET <pierre-alain.toret@protonmail.com>
17 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | 
 3 | Redistribution and use in source and binary forms, with or without
 4 | modification, are permitted provided that the following conditions are
 5 | met:
 6 | 
 7 |    * Redistributions of source code must retain the above copyright
 8 | notice, this list of conditions and the following disclaimer.
 9 |    * Redistributions in binary form must reproduce the above
10 | copyright notice, this list of conditions and the following disclaimer
11 | in the documentation and/or other materials provided with the
12 | distribution.
13 |    * Neither the names of the authors nor the names of the
14 | contributors may be used to endorse or promote products derived from
15 | this software without specific prior written permission.
16 | 
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright (c) 2014 The fileutil authors. All rights reserved.
 2 | # Use of this source code is governed by a BSD-style
 3 | # license that can be found in the LICENSE file.
 4 | 
 5 | .PHONY: all clean editor todo
 6 | 
 7 | all: editor
 8 | 	go vet
 9 | 	golint .
10 | 	go install
11 | 	make todo
12 | 
13 | editor:
14 | 	go fmt
15 | 	go test -i
16 | 	go test
17 | 	go build
18 | 
19 | todo:
20 | 	@grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go || true
21 | 	@grep -n TODO *.go || true
22 | 	@grep -n BUG *.go || true
23 | 	@grep -n println *.go || true
24 | 
25 | clean:
26 | 	@go clean
27 | 	rm -f y.output
28 | 


--------------------------------------------------------------------------------
/README:
--------------------------------------------------------------------------------
 1 | This is a goinstall-able mirror of modified code already published at:
 2 | http://git.nic.cz/redmine/projects/gofileutil/repository
 3 | 
 4 | Packages in this repository:
 5 | 
 6 | Install: $go get github.com/cznic/fileutil
 7 | Godocs: http://godoc.org/github.com/cznic/fileutil
 8 | 
 9 | Install: $go get github.com/cznic/fileutil/storage
10 | Godocs: http://godoc.org/github.com/cznic/fileutil/storage
11 | 
12 | Install: $go get github.com/cznic/fileutil/falloc
13 | Godocs: http://godoc.org/github.com/cznic/fileutil/falloc
14 | 
15 | Install: $go get github.com/cznic/fileutil/hdb
16 | Godocs: http://godoc.org/github.com/cznic/fileutil/hdb
17 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | `github.com/cznic/fileutil` has moved to [`modernc.org/fileutil`](https://godoc.org/modernc.org/fileutil) ([vcs](https://gitlab.com/cznic/fileutil)).
2 | 
3 | Please update your import paths to `modernc.org/fileutil`.
4 | 
5 | This repo is now archived.
6 | 


--------------------------------------------------------------------------------
/all_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package fileutil
 6 | 
 7 | import (
 8 | 	"os"
 9 | 	"path/filepath"
10 | 	"strings"
11 | 	"testing"
12 | )
13 | 
14 | func TestTempFile(t *testing.T) {
15 | 	f, err := TempFile("", "abc", "mno.xyz")
16 | 	if err != nil {
17 | 		t.Fatal(err)
18 | 	}
19 | 
20 | 	n := f.Name()
21 | 	t.Log(n)
22 | 	defer func() {
23 | 		f.Close()
24 | 		os.Remove(n)
25 | 	}()
26 | 
27 | 	base := filepath.Base(n)
28 | 	if base == "abcmno.xyz" {
29 | 		t.Fatal(base)
30 | 	}
31 | 
32 | 	if !strings.HasPrefix(base, "abc") {
33 | 		t.Fatal(base)
34 | 	}
35 | 
36 | 	if !strings.HasSuffix(base, "mno.xyz") {
37 | 		t.Fatal(base)
38 | 	}
39 | }
40 | 


--------------------------------------------------------------------------------
/falloc/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | 
 3 | Redistribution and use in source and binary forms, with or without
 4 | modification, are permitted provided that the following conditions are
 5 | met:
 6 | 
 7 |    * Redistributions of source code must retain the above copyright
 8 | notice, this list of conditions and the following disclaimer.
 9 |    * Redistributions in binary form must reproduce the above
10 | copyright notice, this list of conditions and the following disclaimer
11 | in the documentation and/or other materials provided with the
12 | distribution.
13 |    * Neither the name of CZ.NIC nor the names of its
14 | contributors may be used to endorse or promote products derived from
15 | this software without specific prior written permission.
16 | 
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/falloc/README:
--------------------------------------------------------------------------------
1 | This is a goinstall-able mirror of modified code already published at:
2 | https://git.nic.cz/redmine/projects/gofileutil/repository/show/falloc
3 | 
4 | Install: $go get github.com/cznic/fileutil/falloc
5 | Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/falloc
6 | 


--------------------------------------------------------------------------------
/falloc/all_test.go:
--------------------------------------------------------------------------------
   1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
   2 | // Use of this source code is governed by a BSD-style
   3 | // license that can be found in the LICENSE file.
   4 | 
   5 | // blame: jnml, labs.nic.cz
   6 | 
   7 | package falloc
   8 | 
   9 | import (
  10 | 	"bytes"
  11 | 	"errors"
  12 | 	"flag"
  13 | 	"fmt"
  14 | 	"io/ioutil"
  15 | 	"log"
  16 | 	"math"
  17 | 	"os"
  18 | 	"path/filepath"
  19 | 	"runtime"
  20 | 	"testing"
  21 | 	"time"
  22 | 
  23 | 	"github.com/cznic/fileutil"
  24 | 	"github.com/cznic/fileutil/storage"
  25 | 	"github.com/cznic/mathutil"
  26 | )
  27 | 
  28 | var (
  29 | 	blockFlag      = flag.Uint("block", 256, "block size for some of the dev tests")
  30 | 	cacheTotalFlag = flag.Int64("cachemax", 1<<25, "cache total bytes")
  31 | 	cachedFlag     = flag.Bool("cache", false, "enable caching store")
  32 | 	devFlag        = flag.Bool("dev", false, "enable dev tests")
  33 | 	dropFlag       = flag.Bool("drop", false, "drop system file cache for some of the dev tests before measurement")
  34 | 	fadviseFlag    = flag.Bool("fadvise", false, "hint kernel about random file access")
  35 | 	nFlag          = flag.Int("n", 1, "parameter for some of the dev tests")
  36 | 	probeFlag      = flag.Bool("probe", false, "report store probe statistics")
  37 | 	optGo          = flag.Int("go", 3, "GOMAXPROCS")
  38 | )
  39 | 
  40 | func init() {
  41 | 	flag.Parse()
  42 | 	runtime.GOMAXPROCS(*optGo)
  43 | }
  44 | 
  45 | func temp() (dir, name string) {
  46 | 	dir, err := ioutil.TempDir("", "test-falloc-")
  47 | 	if err != nil {
  48 | 		panic(err)
  49 | 	}
  50 | 
  51 | 	name = filepath.Join(dir, "test.db")
  52 | 	return dir, name
  53 | }
  54 | 
  55 | type balancedAcid struct {
  56 | 	storage.Accessor
  57 | 	nesting int
  58 | }
  59 | 
  60 | func newBalancedAcid(store storage.Accessor) storage.Accessor {
  61 | 	return &balancedAcid{store, 0}
  62 | }
  63 | 
  64 | func (b *balancedAcid) BeginUpdate() error {
  65 | 	if b.nesting < 0 {
  66 | 		return errors.New("BeginUpdate with nesting < 0")
  67 | 	}
  68 | 
  69 | 	b.nesting++
  70 | 	return nil
  71 | }
  72 | 
  73 | func (b *balancedAcid) EndUpdate() error {
  74 | 	if b.nesting <= 0 {
  75 | 		return errors.New("EndUpdate with nesting <= 0")
  76 | 	}
  77 | 
  78 | 	b.nesting--
  79 | 	return nil
  80 | }
  81 | 
  82 | func (b *balancedAcid) Close() error {
  83 | 	if b.nesting != 1 {
  84 | 		return fmt.Errorf("before Close(): nesting %d %p", b.nesting, b)
  85 | 	}
  86 | 
  87 | 	if err := b.Accessor.Close(); err != nil {
  88 | 		return err
  89 | 	}
  90 | 
  91 | 	if b.nesting != 1 {
  92 | 		return fmt.Errorf("after Close(): nesting %d", b.nesting)
  93 | 	}
  94 | 
  95 | 	return nil
  96 | }
  97 | 
  98 | func fopen(fn string) (f *File, err error) {
  99 | 	var store storage.Accessor
 100 | 	if store, err = storage.OpenFile(fn, os.O_RDWR, 0666); err != nil {
 101 | 		return
 102 | 	}
 103 | 
 104 | 	var advise func(int64, int, bool)
 105 | 	if *fadviseFlag {
 106 | 		file := store.(*storage.FileAccessor).File
 107 | 		if err = fileutil.Fadvise(file, 0, 0, fileutil.POSIX_FADV_RANDOM); err != nil {
 108 | 			return
 109 | 		}
 110 | 		advise = func(off int64, len int, write bool) {
 111 | 			if err = fileutil.Fadvise(file, off, off+int64(len), fileutil.POSIX_FADV_DONTNEED); err != nil {
 112 | 				log.Fatal("advisor advise err", err)
 113 | 			}
 114 | 		}
 115 | 	}
 116 | 
 117 | 	var prob *storage.Probe
 118 | 	if *probeFlag {
 119 | 		prob = storage.NewProbe(store, nil)
 120 | 		store = prob
 121 | 	}
 122 | 	if *cachedFlag {
 123 | 		if store, err = storage.NewCache(store, *cacheTotalFlag, advise); err != nil {
 124 | 			return
 125 | 		}
 126 | 
 127 | 		if *probeFlag {
 128 | 			store = storage.NewProbe(store, prob)
 129 | 		}
 130 | 	}
 131 | 	f, err = Open(newBalancedAcid(store))
 132 | 	return
 133 | }
 134 | 
 135 | func fcreate(fn string) (f *File, err error) {
 136 | 	var store storage.Accessor
 137 | 	if store, err = storage.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666); err != nil {
 138 | 		return
 139 | 	}
 140 | 
 141 | 	var advise func(int64, int, bool)
 142 | 	if *fadviseFlag {
 143 | 		file := store.(*storage.FileAccessor).File
 144 | 		if err = fileutil.Fadvise(file, 0, 0, fileutil.POSIX_FADV_RANDOM); err != nil {
 145 | 			return
 146 | 		}
 147 | 		advise = func(off int64, len int, write bool) {
 148 | 			if err = fileutil.Fadvise(file, off, off+int64(len), fileutil.POSIX_FADV_DONTNEED); err != nil {
 149 | 				log.Fatal("advisor advise err", err)
 150 | 			}
 151 | 		}
 152 | 	}
 153 | 
 154 | 	var prob *storage.Probe
 155 | 	if *probeFlag {
 156 | 		prob = storage.NewProbe(store, nil)
 157 | 		store = prob
 158 | 	}
 159 | 	if *cachedFlag {
 160 | 		if store, err = storage.NewCache(store, *cacheTotalFlag, advise); err != nil {
 161 | 			return
 162 | 		}
 163 | 
 164 | 		if *probeFlag {
 165 | 			store = storage.NewProbe(store, prob)
 166 | 		}
 167 | 	}
 168 | 	f, err = New(newBalancedAcid(store))
 169 | 	return
 170 | }
 171 | 
 172 | func probed(t *testing.T, f *File) {
 173 | 	if f == nil {
 174 | 		return
 175 | 	}
 176 | 
 177 | 	dump := func(p *storage.Probe) {
 178 | 		t.Logf("OpsRd %d OpsWr %d BytesRd %d(avg %.1f) BytesWr %d(avg %.1f) SectorsRd %d(%d, +%d, x%.2f) SectorsWr %d(%d, +%d, x%.2f)",
 179 | 			p.OpsRd, p.OpsWr,
 180 | 			p.BytesRd, float64(p.BytesRd)/float64(p.OpsRd),
 181 | 			p.BytesWr, float64(p.BytesWr)/float64(p.OpsWr),
 182 | 			p.SectorsRd,
 183 | 			p.SectorsRd<<9,
 184 | 			p.SectorsRd<<9-p.BytesRd,
 185 | 			float64(p.SectorsRd<<9)/float64(p.BytesRd),
 186 | 			p.SectorsWr,
 187 | 			p.SectorsWr<<9,
 188 | 			p.SectorsWr<<9-p.BytesWr,
 189 | 			float64(p.SectorsWr<<9)/float64(p.BytesWr),
 190 | 		)
 191 | 	}
 192 | 
 193 | 	if ph, ok := f.Accessor().(*storage.Probe); ok {
 194 | 		dump(ph)
 195 | 		if c, ok := ph.Accessor.(*storage.Cache); ok {
 196 | 			if pl, ok := c.Accessor().(*storage.Probe); ok {
 197 | 				dump(pl)
 198 | 			}
 199 | 		}
 200 | 	}
 201 | }
 202 | 
 203 | func (f *File) audit() (usedblocks, totalblocks int64, err error) {
 204 | 	defer func() {
 205 | 		if e := recover(); e != nil {
 206 | 			err = e.(error)
 207 | 		}
 208 | 	}()
 209 | 
 210 | 	fi, err := f.f.Stat()
 211 | 	if err != nil {
 212 | 		panic(err)
 213 | 	}
 214 | 
 215 | 	freemap := map[int64]int64{}
 216 | 	fp := int64(0)
 217 | 	buf := make([]byte, 22)
 218 | 	freeblocks := int64(0)
 219 | 
 220 | 	// linear scan
 221 | 	for fp < fi.Size() {
 222 | 		totalblocks++
 223 | 		typ, size := f.getInfo(fp >> 4)
 224 | 		f.read(buf[:1], fp+size<<4-1)
 225 | 		last := buf[0]
 226 | 		switch {
 227 | 		default:
 228 | 			panic("internal error")
 229 | 		case typ == 0:
 230 | 			if last != 0 {
 231 | 				panic(fmt.Errorf("@%#x used empty, last @%#x: %#x != 0", fp, fp+size<<4-1, last))
 232 | 			}
 233 | 		case typ >= 0x1 && typ <= 0xed:
 234 | 			if last >= 0xfe {
 235 | 				panic(fmt.Errorf("@%#x used short, last @%#x: %#x > 0xfe", fp, fp+size<<4-1, last))
 236 | 			}
 237 | 		case typ >= 0xee && typ <= 0xfb:
 238 | 			if last > 1 {
 239 | 				panic(fmt.Errorf("@%#x used esc short, last @%#x: %#x > 1", fp, fp+size<<4-1, last))
 240 | 			}
 241 | 		case typ == 0xfc:
 242 | 			f.read(buf[:2], fp+1)
 243 | 			switch n := int(buf[0])<<8 + int(buf[1]); {
 244 | 			default:
 245 | 				panic(fmt.Errorf("@%#x used long, illegal content length %#x < 0xee(238)", fp, n))
 246 | 			case n >= 0xee && n <= 0xf0f0:
 247 | 				if last >= 0xfe {
 248 | 					panic(fmt.Errorf("@%#x used long, last @%#x: %#x > 0xfe", fp, fp+size<<4-1, last))
 249 | 				}
 250 | 			case n >= 0xf0f1 && n <= 0xffff:
 251 | 				if last > 1 {
 252 | 					panic(fmt.Errorf("@%#x used esc long, last @%#x: %#x > 1", fp, fp+size<<4-1, last))
 253 | 				}
 254 | 			}
 255 | 		case typ == 0xfd:
 256 | 			if last != 0 {
 257 | 				panic(fmt.Errorf("@%#x reloc, last @%#x: %#x != 0", fp, fp+size<<4-1, last))
 258 | 			}
 259 | 
 260 | 			var target int64
 261 | 			f.read(buf[:7], fp+1)
 262 | 			(*Handle)(&target).Get(buf)
 263 | 			if target >= f.atoms {
 264 | 				panic(fmt.Errorf("@%#x illegal reloc, target %#x > f.atoms(%#x)", fp, target, f.atoms))
 265 | 			}
 266 | 
 267 | 			ttyp, _ := f.getInfo(target)
 268 | 			if ttyp >= 0xfe {
 269 | 				panic(fmt.Errorf("@%#x reloc, points to unused @%#x", fp, target))
 270 | 			}
 271 | 
 272 | 			if ttyp == 0xfd {
 273 | 				panic(fmt.Errorf("@%#x reloc, points to reloc @%#x", fp, target))
 274 | 			}
 275 | 		case typ == 0xfe:
 276 | 			if size < 2 {
 277 | 				panic(fmt.Errorf("@%#x illegal free block, atoms %d < 2", fp, size))
 278 | 			}
 279 | 
 280 | 			if fp>>4 < f.canfree {
 281 | 				panic(fmt.Errorf("@%#x illegal free block @ < f.canfree", fp))
 282 | 			}
 283 | 
 284 | 			f.read(buf[:22], fp)
 285 | 			var prev, next, sz int64
 286 | 			(*Handle)(&prev).Get(buf[1:])
 287 | 			(*Handle)(&next).Get(buf[8:])
 288 | 			f.checkPrevNext(fp, prev, next)
 289 | 			f.read(buf[:7], fp+size<<4-8)
 290 | 			(*Handle)(&sz).Get(buf)
 291 | 			if sz != size {
 292 | 				panic(fmt.Errorf("@%#x mismatch size, %d != %d", fp, sz, size))
 293 | 			}
 294 | 
 295 | 			if last != 0xfe {
 296 | 				panic(fmt.Errorf("@%#x free atom, last @%#x: %#x != 0xff", fp, fp+size<<4-1, last))
 297 | 			}
 298 | 			freemap[fp>>4] = size
 299 | 			freeblocks++
 300 | 		case typ == 0xff:
 301 | 			f.read(buf[:14], fp+1)
 302 | 			var prev, next int64
 303 | 			(*Handle)(&prev).Get(buf)
 304 | 			(*Handle)(&next).Get(buf[7:])
 305 | 			f.checkPrevNext(fp, prev, next)
 306 | 			if last != 0xff {
 307 | 				panic(fmt.Errorf("@%#x free atom, last @%#x: %#x != 0xff", fp, fp+size<<4-1, last))
 308 | 			}
 309 | 			freemap[fp>>4] = size
 310 | 			freeblocks++
 311 | 		}
 312 | 		fp += size << 4
 313 | 	}
 314 | 	usedblocks = totalblocks - freeblocks
 315 | 
 316 | 	// check free table
 317 | 	for size := len(f.freetab) - 1; size > 0; size-- {
 318 | 		var prev, next, fprev int64
 319 | 		this := f.freetab[size]
 320 | 		for this != 0 {
 321 | 			sz, ok := freemap[this]
 322 | 			if !ok {
 323 | 				panic(fmt.Errorf("bad freetab[%d] item @%#x", size, this))
 324 | 			}
 325 | 
 326 | 			delete(freemap, this)
 327 | 
 328 | 			if sz < int64(size) {
 329 | 				panic(fmt.Errorf("bad freetab[%d] item size @%#x %d", size, this, sz))
 330 | 			}
 331 | 
 332 | 			if sz == 1 {
 333 | 				f.read(buf[:15], this<<4)
 334 | 				(*Handle)(&fprev).Get(buf[1:])
 335 | 				if fprev != prev {
 336 | 					panic(fmt.Errorf("bad fprev %#x, exp %#x", fprev, prev))
 337 | 				}
 338 | 
 339 | 				(*Handle)(&next).Get(buf[8:])
 340 | 			} else {
 341 | 				f.read(buf, this<<4)
 342 | 				(*Handle)(&fprev).Get(buf[1:])
 343 | 				if fprev != prev {
 344 | 					panic(fmt.Errorf("bad fprev %#x, exp %#x", fprev, prev))
 345 | 				}
 346 | 				var fsz int64
 347 | 				(*Handle)(&fsz).Get(buf[15:])
 348 | 				if fsz != sz {
 349 | 					panic(fmt.Errorf("bad fsz %d @%#x, exp %#x", fsz, this<<4, sz))
 350 | 				}
 351 | 
 352 | 				(*Handle)(&next).Get(buf[8:])
 353 | 			}
 354 | 
 355 | 			prev, this = this, next
 356 | 		}
 357 | 	}
 358 | 
 359 | 	if n := len(freemap); n != 0 {
 360 | 		for h, s := range freemap {
 361 | 			panic(fmt.Errorf("%d lost free blocks in freemap, e.g. %d free atoms @%#x", n, s, h))
 362 | 		}
 363 | 	}
 364 | 
 365 | 	return
 366 | 
 367 | }
 368 | 
 369 | func (f *File) checkPrevNext(fp, prev, next int64) {
 370 | 	if prev != 0 && prev < f.canfree {
 371 | 		panic(fmt.Errorf("@%#x illegal free atom, prev %#x < f.canfree(%#x)", fp, prev, f.canfree))
 372 | 	}
 373 | 
 374 | 	if prev >= f.atoms {
 375 | 		panic(fmt.Errorf("@%#x illegal free atom, prev %#x > f.atoms", fp, prev))
 376 | 	}
 377 | 
 378 | 	if next != 0 && next < f.canfree {
 379 | 		panic(fmt.Errorf("@%#x illegal free atom, next %#x < f.canfree(%#x)", fp, next, f.canfree))
 380 | 	}
 381 | 
 382 | 	if next >= f.atoms {
 383 | 		panic(fmt.Errorf("@%#x illegal free atom, next %#x > f.atoms", fp, next))
 384 | 	}
 385 | }
 386 | 
 387 | func reaudit(t *testing.T, f *File, fn string) (of *File) {
 388 | 	var err error
 389 | 	if _, _, err := f.audit(); err != nil {
 390 | 		t.Fatal(err)
 391 | 	}
 392 | 
 393 | 	if err := f.Close(); err != nil {
 394 | 		t.Fatal(err)
 395 | 	}
 396 | 
 397 | 	f = nil
 398 | 	runtime.GC()
 399 | 	if of, err = fopen(fn); err != nil {
 400 | 		t.Fatal(err)
 401 | 	}
 402 | 
 403 | 	if _, _, err := of.audit(); err != nil {
 404 | 		t.Fatal(err)
 405 | 	}
 406 | 
 407 | 	return
 408 | }
 409 | 
 410 | func TestCreate(t *testing.T) {
 411 | 	dir, name := temp()
 412 | 	defer os.RemoveAll(dir)
 413 | 
 414 | 	f, err := fcreate(name)
 415 | 	if err != nil {
 416 | 		t.Fatal(err)
 417 | 	}
 418 | 
 419 | 	defer func() {
 420 | 		err := os.Remove(name)
 421 | 		if err != nil {
 422 | 			t.Fatal(err)
 423 | 		}
 424 | 	}()
 425 | 
 426 | 	f.Accessor().Sync()
 427 | 	probed(t, f)
 428 | 	if err = f.Close(); err != nil {
 429 | 		t.Log(f.f.(*balancedAcid).nesting)
 430 | 		t.Fatal(err)
 431 | 	}
 432 | 
 433 | 	b, err := ioutil.ReadFile(name)
 434 | 	if err != nil {
 435 | 		t.Fatal(err)
 436 | 	}
 437 | 
 438 | 	x := b[:16]
 439 | 	if !bytes.Equal(x, hdr) {
 440 | 		t.Fatalf("\n% x\n% x", x, hdr)
 441 | 	}
 442 | 
 443 | 	x = b[16:32]
 444 | 	if !bytes.Equal(x, empty) {
 445 | 		t.Fatalf("\n% x\n% x", x, hdr)
 446 | 	}
 447 | }
 448 | 
 449 | func TestOpen(t *testing.T) {
 450 | 	dir, name := temp()
 451 | 	defer os.RemoveAll(dir)
 452 | 
 453 | 	f, err := fcreate(name)
 454 | 	if err != nil {
 455 | 		t.Fatal(err)
 456 | 	}
 457 | 
 458 | 	defer func() {
 459 | 		probed(t, f)
 460 | 		ec := f.Close()
 461 | 		er := os.Remove(name)
 462 | 		if ec != nil {
 463 | 			t.Fatal(ec)
 464 | 		}
 465 | 
 466 | 		if er != nil {
 467 | 			t.Fatal(er)
 468 | 		}
 469 | 	}()
 470 | 
 471 | 	if err := f.Close(); err != nil {
 472 | 		t.Fatal(err)
 473 | 	}
 474 | 
 475 | 	if f, err = fopen(name); err != nil {
 476 | 		t.Fatal(err)
 477 | 	}
 478 | 
 479 | 	for i, p := range f.freetab {
 480 | 		if p != 0 {
 481 | 			t.Fatal(i+1, p)
 482 | 		}
 483 | 	}
 484 | }
 485 | 
 486 | func alloc(f *File, b []byte) (y int64) {
 487 | 	if h, err := f.Alloc(b); err != nil {
 488 | 		panic(err)
 489 | 	} else {
 490 | 		y = int64(h)
 491 | 	}
 492 | 	return
 493 | }
 494 | 
 495 | func realloc(f *File, atom int64, b []byte, keepHandle bool) (y int64) {
 496 | 	if h, err := f.Realloc(Handle(atom), b, keepHandle); err != nil {
 497 | 		panic(err)
 498 | 	} else {
 499 | 		y = int64(h)
 500 | 	}
 501 | 	return
 502 | }
 503 | 
 504 | func testContentEncodingDecoding(t *testing.T, min, max int) {
 505 | 	dir, name := temp()
 506 | 	defer os.RemoveAll(dir)
 507 | 
 508 | 	f, err := fcreate(name)
 509 | 	if err != nil {
 510 | 		t.Fatal(err)
 511 | 	}
 512 | 
 513 | 	defer func() {
 514 | 		ec := f.Close()
 515 | 		er := os.Remove(name)
 516 | 		if ec != nil {
 517 | 			t.Fatal(ec)
 518 | 		}
 519 | 
 520 | 		if er != nil {
 521 | 			t.Fatal(er)
 522 | 		}
 523 | 	}()
 524 | 
 525 | 	b := make([]byte, max)
 526 | 	r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true)
 527 | 	if err != nil {
 528 | 		t.Fatal(err)
 529 | 	}
 530 | 
 531 | 	blocks := int64(3)
 532 | 	a := make([]int64, 0, 4*(max-min+1))
 533 | 	for cl := min; cl <= max; cl++ {
 534 | 		src := b[:cl]
 535 | 		for i := range src {
 536 | 			b[i] = byte(r.Next())
 537 | 		}
 538 | 		a = append(a, alloc(f, src))
 539 | 		blocks++
 540 | 		if cl == 0 {
 541 | 			continue
 542 | 		}
 543 | 
 544 | 		for i := range src {
 545 | 			b[i] = byte(r.Next())
 546 | 		}
 547 | 		src[cl-1] = 0xfd
 548 | 		a = append(a, alloc(f, src))
 549 | 		blocks++
 550 | 		for i := range src {
 551 | 			b[i] = byte(r.Next())
 552 | 		}
 553 | 		src[cl-1] = 0xfe
 554 | 		a = append(a, alloc(f, src))
 555 | 		blocks++
 556 | 		for i := range src {
 557 | 			b[i] = byte(r.Next())
 558 | 		}
 559 | 		src[cl-1] = 0xff
 560 | 		a = append(a, alloc(f, src))
 561 | 		blocks++
 562 | 	}
 563 | 
 564 | 	f.Accessor().Sync()
 565 | 	probed(t, f)
 566 | 	if err := f.Close(); err != nil {
 567 | 		t.Fatal(err)
 568 | 	}
 569 | 
 570 | 	f = nil
 571 | 	runtime.GC()
 572 | 	if f, err = fopen(name); err != nil {
 573 | 		t.Fatal(err)
 574 | 	}
 575 | 
 576 | 	r.Seek(0)
 577 | 	ai := 0
 578 | 	for cl := min; cl <= max; cl++ {
 579 | 		h := a[ai]
 580 | 		ai++
 581 | 		src := b[:cl]
 582 | 		for i := range src {
 583 | 			b[i] = byte(r.Next())
 584 | 		}
 585 | 		got, _ := f.readUsed(h)
 586 | 		if !bytes.Equal(src, got) {
 587 | 			t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got)
 588 | 		}
 589 | 		if cl == 0 {
 590 | 			continue
 591 | 		}
 592 | 
 593 | 		for i := range src {
 594 | 			b[i] = byte(r.Next())
 595 | 		}
 596 | 		src[cl-1] = 0xfd
 597 | 		h = a[ai]
 598 | 		ai++
 599 | 		got, _ = f.readUsed(h)
 600 | 		if !bytes.Equal(src, got) {
 601 | 			t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got)
 602 | 		}
 603 | 
 604 | 		for i := range src {
 605 | 			b[i] = byte(r.Next())
 606 | 		}
 607 | 		src[cl-1] = 0xfe
 608 | 		h = a[ai]
 609 | 		ai++
 610 | 		got, _ = f.readUsed(h)
 611 | 		if !bytes.Equal(src, got) {
 612 | 			t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got)
 613 | 		}
 614 | 
 615 | 		for i := range src {
 616 | 			b[i] = byte(r.Next())
 617 | 		}
 618 | 		src[cl-1] = 0xff
 619 | 		h = a[ai]
 620 | 		ai++
 621 | 		got, _ = f.readUsed(h)
 622 | 		if !bytes.Equal(src, got) {
 623 | 			t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got)
 624 | 		}
 625 | 	}
 626 | 
 627 | 	auditblocks, _, err := f.audit()
 628 | 	if err != nil {
 629 | 		t.Fatal(err)
 630 | 	}
 631 | 
 632 | 	if auditblocks != blocks {
 633 | 		t.Fatal(auditblocks, blocks)
 634 | 	}
 635 | 
 636 | 	if f = reaudit(t, f, name); err != nil {
 637 | 		t.Fatal(err)
 638 | 	}
 639 | }
 640 | 
 641 | func TestContentEncodingDecoding(t *testing.T) {
 642 | 	testContentEncodingDecoding(t, 0, 1024)
 643 | 	testContentEncodingDecoding(t, 61680-17, 61680)
 644 | }
 645 | 
 646 | type freeItem struct {
 647 | 	size int64
 648 | 	head int64
 649 | }
 650 | 
 651 | func (f *File) reportFree() (report []freeItem) {
 652 | 	for size, head := range f.freetab {
 653 | 		if size != 0 && head != 0 {
 654 | 			report = append(report, freeItem{int64(size), head})
 655 | 		}
 656 | 	}
 657 | 	return
 658 | }
 659 | 
 660 | func free(f *File, h int64) {
 661 | 	if err := f.Free(Handle(h)); err != nil {
 662 | 		panic(err)
 663 | 	}
 664 | }
 665 | 
 666 | func testFreeTail(t *testing.T, b []byte) {
 667 | 	dir, name := temp()
 668 | 	defer os.RemoveAll(dir)
 669 | 
 670 | 	f, err := fcreate(name)
 671 | 	if err != nil {
 672 | 		t.Fatal(err)
 673 | 	}
 674 | 
 675 | 	defer func() {
 676 | 		ec := f.Close()
 677 | 		er := os.Remove(name)
 678 | 		if ec != nil {
 679 | 			t.Fatal(ec)
 680 | 		}
 681 | 
 682 | 		if er != nil {
 683 | 			t.Fatal(er)
 684 | 		}
 685 | 	}()
 686 | 
 687 | 	fs0 := f.atoms
 688 | 	used0, total0, err := f.audit()
 689 | 	if err != nil {
 690 | 		panic(err)
 691 | 	}
 692 | 
 693 | 	if used0 != total0 {
 694 | 		t.Fatal(used0, total0)
 695 | 	}
 696 | 
 697 | 	handle := alloc(f, b)
 698 | 	free(f, handle)
 699 | 	if fs1 := f.atoms; fs1 != fs0 {
 700 | 		t.Fatal(fs1, fs0)
 701 | 	}
 702 | 
 703 | 	if rep := f.reportFree(); len(rep) != 0 {
 704 | 		t.Fatal(rep)
 705 | 	}
 706 | 
 707 | 	if err := f.Close(); err != nil {
 708 | 		t.Fatal(err)
 709 | 	}
 710 | 
 711 | 	f = nil
 712 | 	runtime.GC()
 713 | 	if f, err = fopen(name); err != nil {
 714 | 		t.Fatal(err)
 715 | 	}
 716 | 
 717 | 	used, total, err := f.audit()
 718 | 	if err != nil {
 719 | 		panic(err)
 720 | 	}
 721 | 
 722 | 	if used != used0 {
 723 | 		t.Fatal(used, used0)
 724 | 	}
 725 | 
 726 | 	if total != total0 {
 727 | 		t.Fatal(total, total0)
 728 | 	}
 729 | }
 730 | 
 731 | func TestFreeTail(t *testing.T) {
 732 | 	b := make([]byte, 61680)
 733 | 	for n := 0; n <= 253+16; n++ {
 734 | 		data := b[:n]
 735 | 		testFreeTail(t, data)
 736 | 		if n == 0 {
 737 | 			continue
 738 | 		}
 739 | 
 740 | 		data[n-1] = 0xff
 741 | 		testFreeTail(t, data)
 742 | 		data[n-1] = 0
 743 | 	}
 744 | 
 745 | 	for n := 61680 - 16; n <= 61680; n++ {
 746 | 		data := b[:n]
 747 | 		testFreeTail(t, data)
 748 | 		data[n-1] = 0xff
 749 | 		testFreeTail(t, data)
 750 | 		data[n-1] = 0
 751 | 	}
 752 | }
 753 | 
 754 | func testFreeTail2(t *testing.T, b []byte) {
 755 | 	dir, name := temp()
 756 | 	defer os.RemoveAll(dir)
 757 | 
 758 | 	f, err := fcreate(name)
 759 | 	if err != nil {
 760 | 		t.Fatal(err)
 761 | 	}
 762 | 
 763 | 	defer func() {
 764 | 		ec := f.Close()
 765 | 		er := os.Remove(name)
 766 | 		if ec != nil {
 767 | 			t.Fatal(ec)
 768 | 		}
 769 | 
 770 | 		if er != nil {
 771 | 			t.Fatal(er)
 772 | 		}
 773 | 	}()
 774 | 
 775 | 	fs0 := f.atoms
 776 | 	used0, total0, err := f.audit()
 777 | 	if err != nil {
 778 | 		panic(err)
 779 | 	}
 780 | 
 781 | 	if used0 != total0 {
 782 | 		t.Fatal(used0, total0)
 783 | 	}
 784 | 
 785 | 	handle := alloc(f, b)
 786 | 	handle2 := alloc(f, b)
 787 | 	free(f, handle)
 788 | 	free(f, handle2)
 789 | 	if fs1 := f.atoms; fs1 != fs0 {
 790 | 		t.Fatal(fs1, fs0)
 791 | 	}
 792 | 
 793 | 	if rep := f.reportFree(); len(rep) != 0 {
 794 | 		t.Fatal(rep)
 795 | 	}
 796 | 
 797 | 	if err := f.Close(); err != nil {
 798 | 		t.Fatal(err)
 799 | 	}
 800 | 
 801 | 	f = nil
 802 | 	runtime.GC()
 803 | 	if f, err = fopen(name); err != nil {
 804 | 		t.Fatal(err)
 805 | 	}
 806 | 
 807 | 	used, total, err := f.audit()
 808 | 	if err != nil {
 809 | 		panic(err)
 810 | 	}
 811 | 
 812 | 	if used != used0 {
 813 | 		t.Fatal(used, used0)
 814 | 	}
 815 | 
 816 | 	if total != total0 {
 817 | 		t.Fatal(total, total0)
 818 | 	}
 819 | }
 820 | 
 821 | func TestFreeTail2(t *testing.T) {
 822 | 	b := make([]byte, 61680)
 823 | 	for n := 0; n <= 253+16; n++ {
 824 | 		data := b[:n]
 825 | 		testFreeTail2(t, data)
 826 | 		if n == 0 {
 827 | 			continue
 828 | 		}
 829 | 
 830 | 		data[n-1] = 0xff
 831 | 		testFreeTail2(t, data)
 832 | 		data[n-1] = 0
 833 | 	}
 834 | 
 835 | 	for n := 61680 - 16; n <= 61680; n++ {
 836 | 		data := b[:n]
 837 | 		testFreeTail2(t, data)
 838 | 		data[n-1] = 0xff
 839 | 		testFreeTail2(t, data)
 840 | 		data[n-1] = 0
 841 | 	}
 842 | }
 843 | 
 844 | func testFreeIsolated(t *testing.T, b []byte) {
 845 | 	dir, name := temp()
 846 | 	defer os.RemoveAll(dir)
 847 | 
 848 | 	f, err := fcreate(name)
 849 | 	if err != nil {
 850 | 		t.Fatal(err)
 851 | 	}
 852 | 
 853 | 	defer func() {
 854 | 		ec := f.Close()
 855 | 		er := os.Remove(name)
 856 | 		if ec != nil {
 857 | 			t.Fatal(ec)
 858 | 		}
 859 | 
 860 | 		if er != nil {
 861 | 			t.Fatal(er)
 862 | 		}
 863 | 	}()
 864 | 
 865 | 	rqAtoms := rq2Atoms(len(b))
 866 | 	left := alloc(f, nil)
 867 | 	handle := alloc(f, b)
 868 | 	right := alloc(f, nil)
 869 | 
 870 | 	fs0 := f.atoms
 871 | 	used0, total0, err := f.audit()
 872 | 	if err != nil {
 873 | 		panic(err)
 874 | 	}
 875 | 
 876 | 	if used0 != total0 {
 877 | 		t.Fatal(used0, total0)
 878 | 	}
 879 | 
 880 | 	free(f, handle)
 881 | 	if fs1 := f.atoms; fs1 != fs0 {
 882 | 		t.Fatal(fs1, fs0)
 883 | 	}
 884 | 
 885 | 	rep := f.reportFree()
 886 | 	if len(rep) != 1 {
 887 | 		t.Fatal(rep)
 888 | 	}
 889 | 
 890 | 	if x := rep[0]; x.size != rqAtoms || x.head != handle {
 891 | 		t.Fatal(x)
 892 | 	}
 893 | 
 894 | 	used, total, err := f.audit()
 895 | 	if err != nil {
 896 | 		panic(err)
 897 | 	}
 898 | 
 899 | 	if n, free := f.getSize(left); n != 1 || free {
 900 | 		t.Fatal(n, free)
 901 | 	}
 902 | 
 903 | 	if n, free := f.getSize(right); n != 1 || free {
 904 | 		t.Fatal(n, free)
 905 | 	}
 906 | 
 907 | 	if used != used0-1 {
 908 | 		t.Fatal(used, used0)
 909 | 	}
 910 | 
 911 | 	if total != total0 {
 912 | 		t.Fatal(total, total0)
 913 | 	}
 914 | 
 915 | 	if free := total - used; free != 1 {
 916 | 		t.Fatal(free)
 917 | 	}
 918 | 
 919 | 	// verify persisted file correct
 920 | 	if err := f.Close(); err != nil {
 921 | 		t.Fatal(err)
 922 | 	}
 923 | 
 924 | 	f = nil
 925 | 	runtime.GC()
 926 | 	if f, err = fopen(name); err != nil {
 927 | 		t.Fatal(err)
 928 | 	}
 929 | 
 930 | 	if fs1 := f.atoms; fs1 != fs0 {
 931 | 		t.Fatal(fs1, fs0)
 932 | 	}
 933 | 
 934 | 	rep = f.reportFree()
 935 | 	if len(rep) != 1 {
 936 | 		t.Fatal(rep)
 937 | 	}
 938 | 
 939 | 	if x := rep[0]; x.size != rqAtoms || x.head != handle {
 940 | 		t.Fatal(x)
 941 | 	}
 942 | 
 943 | 	used, total, err = f.audit()
 944 | 	if err != nil {
 945 | 		panic(err)
 946 | 	}
 947 | 
 948 | 	if n, free := f.getSize(left); n != 1 || free {
 949 | 		t.Fatal(n, free)
 950 | 	}
 951 | 
 952 | 	if n, free := f.getSize(right); n != 1 || free {
 953 | 		t.Fatal(n, free)
 954 | 	}
 955 | 
 956 | 	if used != used0-1 {
 957 | 		t.Fatal(used, used0)
 958 | 	}
 959 | 
 960 | 	if total != total0 {
 961 | 		t.Fatal(total, total0)
 962 | 	}
 963 | 
 964 | 	if free := total - used; free != 1 {
 965 | 		t.Fatal(free)
 966 | 	}
 967 | 
 968 | }
 969 | 
 970 | func TestFreeIsolated(t *testing.T) {
 971 | 	b := make([]byte, 61680)
 972 | 	for n := 0; n <= 253+16; n++ {
 973 | 		data := b[:n]
 974 | 		testFreeIsolated(t, data)
 975 | 	}
 976 | 
 977 | 	for n := 61680 - 16; n <= 61680; n++ {
 978 | 		data := b[:n]
 979 | 		testFreeIsolated(t, data)
 980 | 	}
 981 | }
 982 | 
 983 | func testFreeBlockList(t *testing.T, a, b int) {
 984 | 	var h [2]int64
 985 | 
 986 | 	t.Log(a, b)
 987 | 	dir, name := temp()
 988 | 	defer os.RemoveAll(dir)
 989 | 
 990 | 	f, err := fcreate(name)
 991 | 	if err != nil {
 992 | 		t.Fatal(err)
 993 | 	}
 994 | 
 995 | 	defer func() {
 996 | 		if f != nil {
 997 | 			if err := f.Close(); err != nil {
 998 | 				t.Fatal(err)
 999 | 			}
1000 | 		}
1001 | 
1002 | 		f = nil
1003 | 		runtime.GC()
1004 | 		os.Remove(name)
1005 | 	}()
1006 | 
1007 | 	used0, total0, err := f.audit()
1008 | 	if err != nil {
1009 | 		t.Fatal(err)
1010 | 	}
1011 | 
1012 | 	alloc(f, nil)
1013 | 	h[0] = alloc(f, nil)
1014 | 	alloc(f, nil)
1015 | 	h[1] = alloc(f, nil)
1016 | 	alloc(f, nil)
1017 | 
1018 | 	if err := f.Close(); err != nil {
1019 | 		t.Fatal(err)
1020 | 	}
1021 | 
1022 | 	f = nil
1023 | 	runtime.GC()
1024 | 	if f, err = fopen(name); err != nil {
1025 | 		t.Fatal(err)
1026 | 	}
1027 | 
1028 | 	used, total, err := f.audit()
1029 | 	if err != nil {
1030 | 		t.Fatal(err)
1031 | 	}
1032 | 
1033 | 	if used-used0 != 5 || total-total0 != 5 || used != total {
1034 | 		t.Fatal(used0, total0, used, total)
1035 | 	}
1036 | 
1037 | 	free(f, h[a])
1038 | 	free(f, h[b])
1039 | 
1040 | 	used, total, err = f.audit()
1041 | 	if err != nil {
1042 | 		t.Fatal(err)
1043 | 	}
1044 | 
1045 | 	if used-used0 != 3 || total-total0 != 5 || total-used != 2 {
1046 | 		t.Fatal(used0, total0, used, total)
1047 | 	}
1048 | 
1049 | 	if err := f.Close(); err != nil {
1050 | 		t.Fatal(err)
1051 | 	}
1052 | 
1053 | 	f = nil
1054 | 	runtime.GC()
1055 | 	if f, err = fopen(name); err != nil {
1056 | 		t.Fatal(err)
1057 | 	}
1058 | 
1059 | 	used, total, err = f.audit()
1060 | 	if err != nil {
1061 | 		t.Fatal(err)
1062 | 	}
1063 | 
1064 | 	if used-used0 != 3 || total-total0 != 5 || total-used != 2 {
1065 | 		t.Fatal(used0, total0, used, total)
1066 | 	}
1067 | }
1068 | 
1069 | func TestFreeBlockList(t *testing.T) {
1070 | 	testFreeBlockList(t, 0, 1)
1071 | 	testFreeBlockList(t, 1, 0)
1072 | }
1073 | 
1074 | func testFreeBlockList2(t *testing.T, a, b, c int) {
1075 | 	var h [3]int64
1076 | 
1077 | 	dir, name := temp()
1078 | 	defer os.RemoveAll(dir)
1079 | 
1080 | 	f, err := fcreate(name)
1081 | 	if err != nil {
1082 | 		t.Fatal(err)
1083 | 	}
1084 | 
1085 | 	defer func() {
1086 | 		if f != nil {
1087 | 			if err := f.Close(); err != nil {
1088 | 				t.Fatal(err)
1089 | 			}
1090 | 		}
1091 | 
1092 | 		f = nil
1093 | 		runtime.GC()
1094 | 		os.Remove(name)
1095 | 	}()
1096 | 
1097 | 	used0, total0, err := f.audit()
1098 | 	if err != nil {
1099 | 		t.Fatal(err)
1100 | 	}
1101 | 
1102 | 	alloc(f, nil)
1103 | 	h[0] = alloc(f, nil)
1104 | 	alloc(f, nil)
1105 | 	h[1] = alloc(f, nil)
1106 | 	alloc(f, nil)
1107 | 	h[2] = alloc(f, nil)
1108 | 	alloc(f, nil)
1109 | 
1110 | 	if err := f.Close(); err != nil {
1111 | 		t.Fatal(err)
1112 | 	}
1113 | 
1114 | 	f = nil
1115 | 	runtime.GC()
1116 | 	if f, err = fopen(name); err != nil {
1117 | 		t.Fatal(err)
1118 | 	}
1119 | 
1120 | 	used, total, err := f.audit()
1121 | 	if err != nil {
1122 | 		t.Fatal(err)
1123 | 	}
1124 | 
1125 | 	if used-used0 != 7 || total-total0 != 7 || used != total {
1126 | 		t.Fatal(used0, total0, used, total)
1127 | 	}
1128 | 
1129 | 	free(f, h[a])
1130 | 	free(f, h[b])
1131 | 	free(f, h[c])
1132 | 
1133 | 	used, total, err = f.audit()
1134 | 	if err != nil {
1135 | 		t.Fatal(err)
1136 | 	}
1137 | 
1138 | 	if used-used0 != 4 || total-total0 != 7 || total-used != 3 {
1139 | 		t.Fatal(used0, total0, used, total)
1140 | 	}
1141 | 
1142 | 	if err := f.Close(); err != nil {
1143 | 		t.Fatal(err)
1144 | 	}
1145 | 
1146 | 	f = nil
1147 | 	runtime.GC()
1148 | 	if f, err = fopen(name); err != nil {
1149 | 		t.Fatal(err)
1150 | 	}
1151 | 
1152 | 	used, total, err = f.audit()
1153 | 	if err != nil {
1154 | 		t.Fatal(err)
1155 | 	}
1156 | 
1157 | 	if used-used0 != 4 || total-total0 != 7 || total-used != 3 {
1158 | 		t.Fatal(used0, total0, used, total)
1159 | 	}
1160 | }
1161 | 
1162 | func TestFreeBlockList2(t *testing.T) {
1163 | 	testFreeBlockList2(t, 0, 1, 2)
1164 | 	testFreeBlockList2(t, 0, 2, 1)
1165 | 	testFreeBlockList2(t, 1, 0, 2)
1166 | 	testFreeBlockList2(t, 1, 2, 0)
1167 | 	testFreeBlockList2(t, 2, 0, 1)
1168 | 	testFreeBlockList2(t, 2, 1, 0)
1169 | }
1170 | 
1171 | var crng *mathutil.FC32
1172 | 
1173 | func init() {
1174 | 	var err error
1175 | 	if crng, err = mathutil.NewFC32(0, math.MaxInt32, true); err != nil {
1176 | 		panic(err)
1177 | 	}
1178 | }
1179 | 
1180 | func content(b []byte, h int64) (c []byte) {
1181 | 	crng.Seed(h)
1182 | 	crng.Seek(0)
1183 | 	c = b[:crng.Next()%61681]
1184 | 	for i := range c {
1185 | 		c[i] = byte(crng.Next())
1186 | 	}
1187 | 	return
1188 | }
1189 | 
1190 | func testFreeBlockList3(t *testing.T, n, mod int) {
1191 | 	rng, err := mathutil.NewFC32(0, n-1, true)
1192 | 	if err != nil {
1193 | 		t.Fatal(err)
1194 | 	}
1195 | 
1196 | 	dir, name := temp()
1197 | 	defer os.RemoveAll(dir)
1198 | 
1199 | 	f, err := fcreate(name)
1200 | 	if err != nil {
1201 | 		t.Fatal(err)
1202 | 	}
1203 | 
1204 | 	defer func() {
1205 | 		if f != nil {
1206 | 			if err := f.Close(); err != nil {
1207 | 				t.Fatal(err)
1208 | 			}
1209 | 		}
1210 | 
1211 | 		f = nil
1212 | 		runtime.GC()
1213 | 		os.Remove(name)
1214 | 	}()
1215 | 
1216 | 	ha := make([]int64, n)
1217 | 	b := make([]byte, 61680)
1218 | 	for i := range ha {
1219 | 		h := f.atoms
1220 | 		ha[i] = h
1221 | 		c := content(b, h)
1222 | 		if alloc(f, c) != h {
1223 | 			t.Fatal(h)
1224 | 		}
1225 | 	}
1226 | 	f = reaudit(t, f, name)
1227 | 	del := map[int64]bool{}
1228 | 	for _ = range ha {
1229 | 		i := rng.Next()
1230 | 		if i%mod != 0 {
1231 | 			h := ha[i]
1232 | 			free(f, h)
1233 | 			del[h] = true
1234 | 		}
1235 | 	}
1236 | 	f = reaudit(t, f, name)
1237 | 	for _, h := range ha {
1238 | 		if !del[h] {
1239 | 			exp := content(b, h)
1240 | 			got, _ := f.readUsed(h)
1241 | 			if !bytes.Equal(exp, got) {
1242 | 				t.Fatal(len(got), len(exp))
1243 | 			}
1244 | 		}
1245 | 	}
1246 | }
1247 | 
1248 | func TestFreeBlockList3(t *testing.T) {
1249 | 	testFreeBlockList3(t, 111, 1)
1250 | 	testFreeBlockList3(t, 151, 2)
1251 | 	testFreeBlockList3(t, 170, 3)
1252 | 	testFreeBlockList3(t, 170, 4)
1253 | }
1254 | 
1255 | func TestRealloc1(t *testing.T) {
1256 | 	dir, name := temp()
1257 | 	defer os.RemoveAll(dir)
1258 | 
1259 | 	f, err := fcreate(name)
1260 | 	if err != nil {
1261 | 		t.Fatal(err)
1262 | 	}
1263 | 
1264 | 	defer func() {
1265 | 		if f != nil {
1266 | 			if err := f.Close(); err != nil {
1267 | 				t.Fatal(err)
1268 | 			}
1269 | 		}
1270 | 
1271 | 		f = nil
1272 | 		runtime.GC()
1273 | 		os.Remove(name)
1274 | 	}()
1275 | 
1276 | 	b := make([]byte, 61680)
1277 | 	c := content(b, 10)
1278 | 
1279 | 	h10 := alloc(f, nil)
1280 | 	h20 := alloc(f, nil)
1281 | 
1282 | 	used0, total0, err := f.audit()
1283 | 	if err != nil {
1284 | 		t.Fatal(err)
1285 | 	}
1286 | 
1287 | 	exp := c[:15]
1288 | 	if handle := realloc(f, h10, exp, false); handle != h10 {
1289 | 		t.Fatal(handle, h10)
1290 | 	}
1291 | 
1292 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1293 | 		t.Fatal(len(got), len(exp))
1294 | 	}
1295 | 
1296 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1297 | 		t.Fatal(len(got), 0)
1298 | 	}
1299 | 
1300 | 	f = reaudit(t, f, name)
1301 | 
1302 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1303 | 		t.Fatal(len(got), len(exp))
1304 | 	}
1305 | 
1306 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1307 | 		t.Fatal(len(got), 0)
1308 | 	}
1309 | 
1310 | 	used, total, err := f.audit()
1311 | 	if err != nil {
1312 | 		t.Fatal(err)
1313 | 	}
1314 | 
1315 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 0 {
1316 | 		t.Fatal(difused, diftotal, free)
1317 | 	}
1318 | }
1319 | 
1320 | func TestRealloc1Keep(t *testing.T) {
1321 | 	dir, name := temp()
1322 | 	defer os.RemoveAll(dir)
1323 | 
1324 | 	f, err := fcreate(name)
1325 | 	if err != nil {
1326 | 		t.Fatal(err)
1327 | 	}
1328 | 
1329 | 	defer func() {
1330 | 		if f != nil {
1331 | 			if err := f.Close(); err != nil {
1332 | 				t.Fatal(err)
1333 | 			}
1334 | 		}
1335 | 
1336 | 		f = nil
1337 | 		runtime.GC()
1338 | 		os.Remove(name)
1339 | 	}()
1340 | 
1341 | 	b := make([]byte, 61680)
1342 | 	c := content(b, 10)
1343 | 
1344 | 	h10 := alloc(f, nil)
1345 | 	h20 := alloc(f, nil)
1346 | 
1347 | 	used0, total0, err := f.audit()
1348 | 	if err != nil {
1349 | 		t.Fatal(err)
1350 | 	}
1351 | 
1352 | 	exp := c[:15]
1353 | 	if handle := realloc(f, h10, exp, true); handle != h10 {
1354 | 		t.Fatal(handle, h10)
1355 | 	}
1356 | 
1357 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1358 | 		t.Fatal(len(got), len(exp))
1359 | 	}
1360 | 
1361 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1362 | 		t.Fatal(len(got), 0)
1363 | 	}
1364 | 
1365 | 	f = reaudit(t, f, name)
1366 | 
1367 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1368 | 		t.Fatal(len(got), len(exp))
1369 | 	}
1370 | 
1371 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1372 | 		t.Fatal(len(got), 0)
1373 | 	}
1374 | 
1375 | 	used, total, err := f.audit()
1376 | 	if err != nil {
1377 | 		t.Fatal(err)
1378 | 	}
1379 | 
1380 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 0 {
1381 | 		t.Fatal(difused, diftotal, free)
1382 | 	}
1383 | }
1384 | 
1385 | func TestRealloc2(t *testing.T) {
1386 | 	dir, name := temp()
1387 | 	defer os.RemoveAll(dir)
1388 | 
1389 | 	f, err := fcreate(name)
1390 | 	if err != nil {
1391 | 		t.Fatal(err)
1392 | 	}
1393 | 
1394 | 	defer func() {
1395 | 		if f != nil {
1396 | 			if err := f.Close(); err != nil {
1397 | 				t.Fatal(err)
1398 | 			}
1399 | 		}
1400 | 
1401 | 		f = nil
1402 | 		runtime.GC()
1403 | 		os.Remove(name)
1404 | 	}()
1405 | 
1406 | 	b := make([]byte, 61680)
1407 | 	c := content(b, 10)
1408 | 
1409 | 	h10 := alloc(f, c[:31])
1410 | 	h20 := alloc(f, nil)
1411 | 
1412 | 	used0, total0, err := f.audit()
1413 | 	if err != nil {
1414 | 		t.Fatal(err)
1415 | 	}
1416 | 
1417 | 	exp := c[:15]
1418 | 	if handle := realloc(f, h10, exp, false); handle != h10 {
1419 | 		t.Fatal(handle, h10)
1420 | 	}
1421 | 
1422 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1423 | 		t.Fatal(len(got), len(exp))
1424 | 	}
1425 | 
1426 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1427 | 		t.Fatal(len(got), 0)
1428 | 	}
1429 | 
1430 | 	f = reaudit(t, f, name)
1431 | 
1432 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1433 | 		t.Fatal(len(got), len(exp))
1434 | 	}
1435 | 
1436 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1437 | 		t.Fatal(len(got), 0)
1438 | 	}
1439 | 
1440 | 	used, total, err := f.audit()
1441 | 	if err != nil {
1442 | 		t.Fatal(err)
1443 | 	}
1444 | 
1445 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 {
1446 | 		t.Fatal(difused, diftotal, free)
1447 | 	}
1448 | }
1449 | 
1450 | func TestRealloc2Keep(t *testing.T) {
1451 | 	dir, name := temp()
1452 | 	defer os.RemoveAll(dir)
1453 | 
1454 | 	f, err := fcreate(name)
1455 | 	if err != nil {
1456 | 		t.Fatal(err)
1457 | 	}
1458 | 
1459 | 	defer func() {
1460 | 		if f != nil {
1461 | 			if err := f.Close(); err != nil {
1462 | 				t.Fatal(err)
1463 | 			}
1464 | 		}
1465 | 
1466 | 		f = nil
1467 | 		runtime.GC()
1468 | 		os.Remove(name)
1469 | 	}()
1470 | 
1471 | 	b := make([]byte, 61680)
1472 | 	c := content(b, 10)
1473 | 
1474 | 	h10 := alloc(f, c[:31])
1475 | 	h20 := alloc(f, nil)
1476 | 
1477 | 	used0, total0, err := f.audit()
1478 | 	if err != nil {
1479 | 		t.Fatal(err)
1480 | 	}
1481 | 
1482 | 	exp := c[:15]
1483 | 	if handle := realloc(f, h10, exp, true); handle != h10 {
1484 | 		t.Fatal(handle, h10)
1485 | 	}
1486 | 
1487 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1488 | 		t.Fatal(len(got), len(exp))
1489 | 	}
1490 | 
1491 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1492 | 		t.Fatal(len(got), 0)
1493 | 	}
1494 | 
1495 | 	f = reaudit(t, f, name)
1496 | 
1497 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
1498 | 		t.Fatal(len(got), len(exp))
1499 | 	}
1500 | 
1501 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1502 | 		t.Fatal(len(got), 0)
1503 | 	}
1504 | 
1505 | 	used, total, err := f.audit()
1506 | 	if err != nil {
1507 | 		t.Fatal(err)
1508 | 	}
1509 | 
1510 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 {
1511 | 		t.Fatal(difused, diftotal, free)
1512 | 	}
1513 | }
1514 | 
1515 | func TestRealloc3(t *testing.T) {
1516 | 	dir, name := temp()
1517 | 	defer os.RemoveAll(dir)
1518 | 
1519 | 	f, err := fcreate(name)
1520 | 	if err != nil {
1521 | 		t.Fatal(err)
1522 | 	}
1523 | 
1524 | 	defer func() {
1525 | 		if f != nil {
1526 | 			if err := f.Close(); err != nil {
1527 | 				t.Fatal(err)
1528 | 			}
1529 | 		}
1530 | 
1531 | 		f = nil
1532 | 		runtime.GC()
1533 | 		os.Remove(name)
1534 | 	}()
1535 | 
1536 | 	b := make([]byte, 61680)
1537 | 	c := content(b, 10)
1538 | 
1539 | 	h10 := alloc(f, nil)
1540 | 	h20 := alloc(f, nil)
1541 | 
1542 | 	used0, total0, err := f.audit()
1543 | 	if err != nil {
1544 | 		t.Fatal(err)
1545 | 	}
1546 | 
1547 | 	exp := c[:31]
1548 | 	var handle int64
1549 | 	if handle = realloc(f, h10, exp, false); handle == h10 {
1550 | 		t.Fatal(handle, h10)
1551 | 	}
1552 | 
1553 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1554 | 		t.Fatal(len(got), len(exp))
1555 | 	}
1556 | 
1557 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1558 | 		t.Fatal(len(got), 0)
1559 | 	}
1560 | 
1561 | 	f = reaudit(t, f, name)
1562 | 
1563 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1564 | 		t.Fatal(len(got), len(exp))
1565 | 	}
1566 | 
1567 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1568 | 		t.Fatal(len(got), 0)
1569 | 	}
1570 | 
1571 | 	used, total, err := f.audit()
1572 | 	if err != nil {
1573 | 		t.Fatal(err)
1574 | 	}
1575 | 
1576 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 {
1577 | 		t.Fatal(difused, diftotal, free)
1578 | 	}
1579 | }
1580 | 
1581 | func TestRealloc3Keep(t *testing.T) {
1582 | 	dir, name := temp()
1583 | 	defer os.RemoveAll(dir)
1584 | 
1585 | 	f, err := fcreate(name)
1586 | 	if err != nil {
1587 | 		t.Fatal(err)
1588 | 	}
1589 | 
1590 | 	defer func() {
1591 | 		if f != nil {
1592 | 			if err := f.Close(); err != nil {
1593 | 				t.Fatal(err)
1594 | 			}
1595 | 		}
1596 | 
1597 | 		f = nil
1598 | 		runtime.GC()
1599 | 		os.Remove(name)
1600 | 	}()
1601 | 
1602 | 	b := make([]byte, 61680)
1603 | 	c := content(b, 10)
1604 | 
1605 | 	h10 := alloc(f, nil)
1606 | 	h20 := alloc(f, nil)
1607 | 
1608 | 	used0, total0, err := f.audit()
1609 | 	if err != nil {
1610 | 		t.Fatal(err)
1611 | 	}
1612 | 
1613 | 	exp := c[:31]
1614 | 	var handle int64
1615 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
1616 | 		t.Fatal(handle, h10)
1617 | 	}
1618 | 
1619 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1620 | 		t.Fatal(len(got), len(exp))
1621 | 	}
1622 | 
1623 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1624 | 		t.Fatal(len(got), 0)
1625 | 	}
1626 | 
1627 | 	f = reaudit(t, f, name)
1628 | 
1629 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1630 | 		t.Fatal(len(got), len(exp))
1631 | 	}
1632 | 
1633 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1634 | 		t.Fatal(len(got), 0)
1635 | 	}
1636 | 
1637 | 	used, total, err := f.audit()
1638 | 	if err != nil {
1639 | 		t.Fatal(err)
1640 | 	}
1641 | 
1642 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 1 || free != 0 {
1643 | 		t.Fatal(difused, diftotal, free)
1644 | 	}
1645 | }
1646 | 
1647 | func TestRealloc4Keep(t *testing.T) {
1648 | 	dir, name := temp()
1649 | 	defer os.RemoveAll(dir)
1650 | 
1651 | 	f, err := fcreate(name)
1652 | 	if err != nil {
1653 | 		t.Fatal(err)
1654 | 	}
1655 | 
1656 | 	defer func() {
1657 | 		if f != nil {
1658 | 			if err := f.Close(); err != nil {
1659 | 				t.Fatal(err)
1660 | 			}
1661 | 		}
1662 | 
1663 | 		f = nil
1664 | 		runtime.GC()
1665 | 		os.Remove(name)
1666 | 	}()
1667 | 
1668 | 	b := make([]byte, 61680)
1669 | 	c := content(b, 10)
1670 | 
1671 | 	h10 := alloc(f, c[:31])
1672 | 	h20 := alloc(f, nil)
1673 | 
1674 | 	used0, total0, err := f.audit()
1675 | 	if err != nil {
1676 | 		t.Fatal(err)
1677 | 	}
1678 | 
1679 | 	exp := c[:47]
1680 | 	var handle int64
1681 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
1682 | 		t.Fatal(handle, h10)
1683 | 	}
1684 | 
1685 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1686 | 		t.Fatal(len(got), len(exp))
1687 | 	}
1688 | 
1689 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1690 | 		t.Fatal(len(got), 0)
1691 | 	}
1692 | 
1693 | 	f = reaudit(t, f, name)
1694 | 
1695 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1696 | 		t.Fatal(len(got), len(exp))
1697 | 	}
1698 | 
1699 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1700 | 		t.Fatal(len(got), 0)
1701 | 	}
1702 | 
1703 | 	used, total, err := f.audit()
1704 | 	if err != nil {
1705 | 		t.Fatal(err)
1706 | 	}
1707 | 
1708 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 2 || free != 1 {
1709 | 		t.Fatal(difused, diftotal, free)
1710 | 	}
1711 | }
1712 | 
1713 | func TestRealloc5(t *testing.T) {
1714 | 	dir, name := temp()
1715 | 	defer os.RemoveAll(dir)
1716 | 
1717 | 	f, err := fcreate(name)
1718 | 	if err != nil {
1719 | 		t.Fatal(err)
1720 | 	}
1721 | 
1722 | 	defer func() {
1723 | 		if f != nil {
1724 | 			if err := f.Close(); err != nil {
1725 | 				t.Fatal(err)
1726 | 			}
1727 | 		}
1728 | 
1729 | 		f = nil
1730 | 		runtime.GC()
1731 | 		os.Remove(name)
1732 | 	}()
1733 | 
1734 | 	b := make([]byte, 61680)
1735 | 	c := content(b, 10)
1736 | 
1737 | 	h10 := alloc(f, nil)
1738 | 	h15 := alloc(f, nil)
1739 | 	h20 := alloc(f, nil)
1740 | 
1741 | 	used0, total0, err := f.audit()
1742 | 	if err != nil {
1743 | 		t.Fatal(err)
1744 | 	}
1745 | 
1746 | 	free(f, h15)
1747 | 	exp := c[:31]
1748 | 	var handle int64
1749 | 	if handle = realloc(f, h10, exp, false); handle != h10 {
1750 | 		t.Fatal(handle, h10)
1751 | 	}
1752 | 
1753 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1754 | 		t.Fatal(len(got), len(exp))
1755 | 	}
1756 | 
1757 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1758 | 		t.Fatal(len(got), 0)
1759 | 	}
1760 | 
1761 | 	f = reaudit(t, f, name)
1762 | 
1763 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1764 | 		t.Fatal(len(got), len(exp))
1765 | 	}
1766 | 
1767 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1768 | 		t.Fatal(len(got), 0)
1769 | 	}
1770 | 
1771 | 	used, total, err := f.audit()
1772 | 	if err != nil {
1773 | 		t.Fatal(err)
1774 | 	}
1775 | 
1776 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 {
1777 | 		t.Fatal(difused, diftotal, free)
1778 | 	}
1779 | }
1780 | 
1781 | func TestRealloc5Keep(t *testing.T) {
1782 | 	dir, name := temp()
1783 | 	defer os.RemoveAll(dir)
1784 | 
1785 | 	f, err := fcreate(name)
1786 | 	if err != nil {
1787 | 		t.Fatal(err)
1788 | 	}
1789 | 
1790 | 	defer func() {
1791 | 		if f != nil {
1792 | 			if err := f.Close(); err != nil {
1793 | 				t.Fatal(err)
1794 | 			}
1795 | 		}
1796 | 
1797 | 		f = nil
1798 | 		runtime.GC()
1799 | 		os.Remove(name)
1800 | 	}()
1801 | 
1802 | 	b := make([]byte, 61680)
1803 | 	c := content(b, 10)
1804 | 
1805 | 	h10 := alloc(f, nil)
1806 | 	h15 := alloc(f, nil)
1807 | 	h20 := alloc(f, nil)
1808 | 
1809 | 	used0, total0, err := f.audit()
1810 | 	if err != nil {
1811 | 		t.Fatal(err)
1812 | 	}
1813 | 
1814 | 	free(f, h15)
1815 | 	exp := c[:31]
1816 | 	var handle int64
1817 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
1818 | 		t.Fatal(handle, h10)
1819 | 	}
1820 | 
1821 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1822 | 		t.Fatal(len(got), len(exp))
1823 | 	}
1824 | 
1825 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1826 | 		t.Fatal(len(got), 0)
1827 | 	}
1828 | 
1829 | 	f = reaudit(t, f, name)
1830 | 
1831 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1832 | 		t.Fatal(len(got), len(exp))
1833 | 	}
1834 | 
1835 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1836 | 		t.Fatal(len(got), 0)
1837 | 	}
1838 | 
1839 | 	used, total, err := f.audit()
1840 | 	if err != nil {
1841 | 		t.Fatal(err)
1842 | 	}
1843 | 
1844 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 {
1845 | 		t.Fatal(difused, diftotal, free)
1846 | 	}
1847 | }
1848 | 
1849 | func TestRealloc6(t *testing.T) {
1850 | 	dir, name := temp()
1851 | 	defer os.RemoveAll(dir)
1852 | 
1853 | 	f, err := fcreate(name)
1854 | 	if err != nil {
1855 | 		t.Fatal(err)
1856 | 	}
1857 | 
1858 | 	defer func() {
1859 | 		if f != nil {
1860 | 			if err := f.Close(); err != nil {
1861 | 				t.Fatal(err)
1862 | 			}
1863 | 		}
1864 | 
1865 | 		f = nil
1866 | 		runtime.GC()
1867 | 		os.Remove(name)
1868 | 	}()
1869 | 
1870 | 	b := make([]byte, 61680)
1871 | 	c := content(b, 10)
1872 | 
1873 | 	h10 := alloc(f, nil)
1874 | 	h15 := alloc(f, c[:31])
1875 | 	h20 := alloc(f, nil)
1876 | 
1877 | 	used0, total0, err := f.audit()
1878 | 	if err != nil {
1879 | 		t.Fatal(err)
1880 | 	}
1881 | 
1882 | 	free(f, h15)
1883 | 	exp := c[:31]
1884 | 	var handle int64
1885 | 	if handle = realloc(f, h10, exp, false); handle != h10 {
1886 | 		t.Fatal(handle, h10)
1887 | 	}
1888 | 
1889 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1890 | 		t.Fatal(len(got), len(exp))
1891 | 	}
1892 | 
1893 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1894 | 		t.Fatal(len(got), 0)
1895 | 	}
1896 | 
1897 | 	f = reaudit(t, f, name)
1898 | 
1899 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1900 | 		t.Fatal(len(got), len(exp))
1901 | 	}
1902 | 
1903 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1904 | 		t.Fatal(len(got), 0)
1905 | 	}
1906 | 
1907 | 	used, total, err := f.audit()
1908 | 	if err != nil {
1909 | 		t.Fatal(err)
1910 | 	}
1911 | 
1912 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != 0 || free != 1 {
1913 | 		t.Fatal(difused, diftotal, free)
1914 | 	}
1915 | }
1916 | 
1917 | func TestRealloc6Keep(t *testing.T) {
1918 | 	dir, name := temp()
1919 | 	defer os.RemoveAll(dir)
1920 | 
1921 | 	f, err := fcreate(name)
1922 | 	if err != nil {
1923 | 		t.Fatal(err)
1924 | 	}
1925 | 
1926 | 	defer func() {
1927 | 		if f != nil {
1928 | 			if err := f.Close(); err != nil {
1929 | 				t.Fatal(err)
1930 | 			}
1931 | 		}
1932 | 
1933 | 		f = nil
1934 | 		runtime.GC()
1935 | 		os.Remove(name)
1936 | 	}()
1937 | 
1938 | 	b := make([]byte, 61680)
1939 | 	c := content(b, 10)
1940 | 
1941 | 	h10 := alloc(f, nil)
1942 | 	h15 := alloc(f, c[:31])
1943 | 	h20 := alloc(f, nil)
1944 | 
1945 | 	used0, total0, err := f.audit()
1946 | 	if err != nil {
1947 | 		t.Fatal(err)
1948 | 	}
1949 | 
1950 | 	free(f, h15)
1951 | 	exp := c[:31]
1952 | 	var handle int64
1953 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
1954 | 		t.Fatal(handle, h10)
1955 | 	}
1956 | 
1957 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1958 | 		t.Fatal(len(got), len(exp))
1959 | 	}
1960 | 
1961 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1962 | 		t.Fatal(len(got), 0)
1963 | 	}
1964 | 
1965 | 	f = reaudit(t, f, name)
1966 | 
1967 | 	if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) {
1968 | 		t.Fatal(len(got), len(exp))
1969 | 	}
1970 | 
1971 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
1972 | 		t.Fatal(len(got), 0)
1973 | 	}
1974 | 
1975 | 	used, total, err := f.audit()
1976 | 	if err != nil {
1977 | 		t.Fatal(err)
1978 | 	}
1979 | 
1980 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != 0 || free != 1 {
1981 | 		t.Fatal(difused, diftotal, free)
1982 | 	}
1983 | }
1984 | 
1985 | func TestRelocRealloc1(t *testing.T) {
1986 | 	dir, name := temp()
1987 | 	defer os.RemoveAll(dir)
1988 | 
1989 | 	f, err := fcreate(name)
1990 | 	if err != nil {
1991 | 		t.Fatal(err)
1992 | 	}
1993 | 
1994 | 	defer func() {
1995 | 		if f != nil {
1996 | 			if err := f.Close(); err != nil {
1997 | 				t.Fatal(err)
1998 | 			}
1999 | 		}
2000 | 
2001 | 		f = nil
2002 | 		runtime.GC()
2003 | 		os.Remove(name)
2004 | 	}()
2005 | 
2006 | 	b := make([]byte, 61680)
2007 | 
2008 | 	h10 := alloc(f, nil)
2009 | 	h20 := alloc(f, nil)
2010 | 	var handle int64
2011 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2012 | 		t.Fatal(handle, h10)
2013 | 	}
2014 | 
2015 | 	used0, total0, err := f.audit() // c+3, c+3
2016 | 	if err != nil {
2017 | 		t.Fatal(err)
2018 | 	}
2019 | 
2020 | 	c := content(b, 10)
2021 | 	exp := c[:15]
2022 | 	if handle = realloc(f, h10, exp, false); handle != h10 {
2023 | 		t.Fatal(handle, h10)
2024 | 	}
2025 | 
2026 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2027 | 		t.Fatal(len(got), len(exp))
2028 | 	}
2029 | 
2030 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
2031 | 		t.Fatal(len(got), 0)
2032 | 	}
2033 | 
2034 | 	f = reaudit(t, f, name)
2035 | 
2036 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2037 | 		t.Fatal(len(got), len(exp))
2038 | 	}
2039 | 
2040 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
2041 | 		t.Fatal(len(got), 0)
2042 | 	}
2043 | 
2044 | 	used, total, err := f.audit() // c+2, c+2
2045 | 	if err != nil {
2046 | 		t.Fatal(err)
2047 | 	}
2048 | 
2049 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 {
2050 | 		t.Fatal(difused, diftotal, free)
2051 | 	}
2052 | }
2053 | 
2054 | func TestRelocRealloc1Keep(t *testing.T) {
2055 | 	dir, name := temp()
2056 | 	defer os.RemoveAll(dir)
2057 | 
2058 | 	f, err := fcreate(name)
2059 | 	if err != nil {
2060 | 		t.Fatal(err)
2061 | 	}
2062 | 
2063 | 	defer func() {
2064 | 		if f != nil {
2065 | 			if err := f.Close(); err != nil {
2066 | 				t.Fatal(err)
2067 | 			}
2068 | 		}
2069 | 
2070 | 		f = nil
2071 | 		runtime.GC()
2072 | 		os.Remove(name)
2073 | 	}()
2074 | 
2075 | 	b := make([]byte, 61680)
2076 | 
2077 | 	h10 := alloc(f, nil)
2078 | 	h20 := alloc(f, nil)
2079 | 	var handle int64
2080 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2081 | 		t.Fatal(handle, h10)
2082 | 	}
2083 | 
2084 | 	used0, total0, err := f.audit() // c+3, c+3
2085 | 	if err != nil {
2086 | 		t.Fatal(err)
2087 | 	}
2088 | 
2089 | 	c := content(b, 10)
2090 | 	exp := c[:15]
2091 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
2092 | 		t.Fatal(handle, h10)
2093 | 	}
2094 | 
2095 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2096 | 		t.Fatal(len(got), len(exp))
2097 | 	}
2098 | 
2099 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
2100 | 		t.Fatal(len(got), 0)
2101 | 	}
2102 | 
2103 | 	f = reaudit(t, f, name)
2104 | 
2105 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2106 | 		t.Fatal(len(got), len(exp))
2107 | 	}
2108 | 
2109 | 	if got, _ := f.readUsed(h20); len(got) != 0 {
2110 | 		t.Fatal(len(got), 0)
2111 | 	}
2112 | 
2113 | 	used, total, err := f.audit() // c+2, c+2
2114 | 	if err != nil {
2115 | 		t.Fatal(err)
2116 | 	}
2117 | 
2118 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 {
2119 | 		t.Fatal(difused, diftotal, free)
2120 | 	}
2121 | }
2122 | 
2123 | func TestRelocRealloc2(t *testing.T) {
2124 | 	dir, name := temp()
2125 | 	defer os.RemoveAll(dir)
2126 | 
2127 | 	f, err := fcreate(name)
2128 | 	if err != nil {
2129 | 		t.Fatal(err)
2130 | 	}
2131 | 
2132 | 	defer func() {
2133 | 		if f != nil {
2134 | 			if err := f.Close(); err != nil {
2135 | 				t.Fatal(err)
2136 | 			}
2137 | 		}
2138 | 
2139 | 		f = nil
2140 | 		runtime.GC()
2141 | 		os.Remove(name)
2142 | 	}()
2143 | 
2144 | 	b := make([]byte, 61680)
2145 | 
2146 | 	h10 := alloc(f, nil)
2147 | 	h20 := alloc(f, nil)
2148 | 	var handle int64
2149 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2150 | 		t.Fatal(handle, h10)
2151 | 	}
2152 | 
2153 | 	free(f, h20)
2154 | 
2155 | 	used0, total0, err := f.audit() // c+2, c+3
2156 | 	if err != nil {
2157 | 		t.Fatal(err)
2158 | 	}
2159 | 
2160 | 	c := content(b, 10)
2161 | 	exp := c[:31]
2162 | 	if handle = realloc(f, h10, exp, false); handle != h10 {
2163 | 		t.Fatal(handle, h10)
2164 | 	}
2165 | 
2166 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2167 | 		t.Fatal(len(got), len(exp))
2168 | 	}
2169 | 
2170 | 	f = reaudit(t, f, name)
2171 | 
2172 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2173 | 		t.Fatal(len(got), len(exp))
2174 | 	}
2175 | 
2176 | 	used, total, err := f.audit() // c+1, c+1
2177 | 	if err != nil {
2178 | 		t.Fatal(err)
2179 | 	}
2180 | 
2181 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 {
2182 | 		t.Fatal(difused, diftotal, free)
2183 | 	}
2184 | }
2185 | 
2186 | func TestRelocRealloc2Keep(t *testing.T) {
2187 | 	dir, name := temp()
2188 | 	defer os.RemoveAll(dir)
2189 | 
2190 | 	f, err := fcreate(name)
2191 | 	if err != nil {
2192 | 		t.Fatal(err)
2193 | 	}
2194 | 
2195 | 	defer func() {
2196 | 		if f != nil {
2197 | 			if err := f.Close(); err != nil {
2198 | 				t.Fatal(err)
2199 | 			}
2200 | 		}
2201 | 
2202 | 		f = nil
2203 | 		runtime.GC()
2204 | 		os.Remove(name)
2205 | 	}()
2206 | 
2207 | 	b := make([]byte, 61680)
2208 | 
2209 | 	h10 := alloc(f, nil)
2210 | 	h20 := alloc(f, nil)
2211 | 	var handle int64
2212 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2213 | 		t.Fatal(handle, h10)
2214 | 	}
2215 | 
2216 | 	free(f, h20)
2217 | 
2218 | 	used0, total0, err := f.audit() // c+2, c+3
2219 | 	if err != nil {
2220 | 		t.Fatal(err)
2221 | 	}
2222 | 
2223 | 	c := content(b, 10)
2224 | 	exp := c[:31]
2225 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
2226 | 		t.Fatal(handle, h10)
2227 | 	}
2228 | 
2229 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2230 | 		t.Fatal(len(got), len(exp))
2231 | 	}
2232 | 
2233 | 	f = reaudit(t, f, name)
2234 | 
2235 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2236 | 		t.Fatal(len(got), len(exp))
2237 | 	}
2238 | 
2239 | 	used, total, err := f.audit() // c+1, c+1
2240 | 	if err != nil {
2241 | 		t.Fatal(err)
2242 | 	}
2243 | 
2244 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 {
2245 | 		t.Fatal(difused, diftotal, free)
2246 | 	}
2247 | }
2248 | 
2249 | func TestRelocRealloc3(t *testing.T) {
2250 | 	dir, name := temp()
2251 | 	defer os.RemoveAll(dir)
2252 | 
2253 | 	f, err := fcreate(name)
2254 | 	if err != nil {
2255 | 		t.Fatal(err)
2256 | 	}
2257 | 
2258 | 	defer func() {
2259 | 		if f != nil {
2260 | 			if err := f.Close(); err != nil {
2261 | 				t.Fatal(err)
2262 | 			}
2263 | 		}
2264 | 
2265 | 		f = nil
2266 | 		runtime.GC()
2267 | 		os.Remove(name)
2268 | 	}()
2269 | 
2270 | 	b := make([]byte, 61680)
2271 | 
2272 | 	h10 := alloc(f, nil)
2273 | 	h20 := alloc(f, b[:31])
2274 | 	var handle int64
2275 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2276 | 		t.Fatal(handle, h10)
2277 | 	}
2278 | 
2279 | 	free(f, h20)
2280 | 
2281 | 	used0, total0, err := f.audit() // c+2, c+3
2282 | 	if err != nil {
2283 | 		t.Fatal(err)
2284 | 	}
2285 | 
2286 | 	c := content(b, 10)
2287 | 	exp := c[:31]
2288 | 	if handle = realloc(f, h10, exp, false); handle != h10 {
2289 | 		t.Fatal(handle, h10)
2290 | 	}
2291 | 
2292 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2293 | 		t.Fatal(len(got), len(exp))
2294 | 	}
2295 | 
2296 | 	f = reaudit(t, f, name)
2297 | 
2298 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2299 | 		t.Fatal(len(got), len(exp))
2300 | 	}
2301 | 
2302 | 	used, total, err := f.audit() // c+1, c+1
2303 | 	if err != nil {
2304 | 		t.Fatal(err)
2305 | 	}
2306 | 
2307 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 {
2308 | 		t.Fatal(difused, diftotal, free)
2309 | 	}
2310 | }
2311 | 
2312 | func TestRelocRealloc3Keep(t *testing.T) {
2313 | 	dir, name := temp()
2314 | 	defer os.RemoveAll(dir)
2315 | 
2316 | 	f, err := fcreate(name)
2317 | 	if err != nil {
2318 | 		t.Fatal(err)
2319 | 	}
2320 | 
2321 | 	defer func() {
2322 | 		if f != nil {
2323 | 			if err := f.Close(); err != nil {
2324 | 				t.Fatal(err)
2325 | 			}
2326 | 		}
2327 | 
2328 | 		f = nil
2329 | 		runtime.GC()
2330 | 		os.Remove(name)
2331 | 	}()
2332 | 
2333 | 	b := make([]byte, 61680)
2334 | 
2335 | 	h10 := alloc(f, nil)
2336 | 	h20 := alloc(f, b[:31])
2337 | 	var handle int64
2338 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2339 | 		t.Fatal(handle, h10)
2340 | 	}
2341 | 
2342 | 	free(f, h20)
2343 | 
2344 | 	used0, total0, err := f.audit() // c+2, c+3
2345 | 	if err != nil {
2346 | 		t.Fatal(err)
2347 | 	}
2348 | 
2349 | 	c := content(b, 10)
2350 | 	exp := c[:31]
2351 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
2352 | 		t.Fatal(handle, h10)
2353 | 	}
2354 | 
2355 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2356 | 		t.Fatal(len(got), len(exp))
2357 | 	}
2358 | 
2359 | 	f = reaudit(t, f, name)
2360 | 
2361 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2362 | 		t.Fatal(len(got), len(exp))
2363 | 	}
2364 | 
2365 | 	used, total, err := f.audit() // c+1, c+1
2366 | 	if err != nil {
2367 | 		t.Fatal(err)
2368 | 	}
2369 | 
2370 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 {
2371 | 		t.Fatal(difused, diftotal, free)
2372 | 	}
2373 | }
2374 | 
2375 | func TestRelocRealloc4(t *testing.T) {
2376 | 	dir, name := temp()
2377 | 	defer os.RemoveAll(dir)
2378 | 
2379 | 	f, err := fcreate(name)
2380 | 	if err != nil {
2381 | 		t.Fatal(err)
2382 | 	}
2383 | 
2384 | 	defer func() {
2385 | 		if f != nil {
2386 | 			if err := f.Close(); err != nil {
2387 | 				t.Fatal(err)
2388 | 			}
2389 | 		}
2390 | 
2391 | 		f = nil
2392 | 		runtime.GC()
2393 | 		os.Remove(name)
2394 | 	}()
2395 | 
2396 | 	b := make([]byte, 61680)
2397 | 
2398 | 	h10 := alloc(f, nil)
2399 | 	_ = alloc(f, nil)
2400 | 	var handle int64
2401 | 	if handle = realloc(f, h10, b[:47], true); handle != h10 {
2402 | 		t.Fatal(handle, h10)
2403 | 	}
2404 | 
2405 | 	_ = alloc(f, nil)
2406 | 
2407 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2408 | 		t.Fatal(handle, h10)
2409 | 	}
2410 | 
2411 | 	used0, total0, err := f.audit() // c+4, c+5
2412 | 	if err != nil {
2413 | 		t.Fatal(err)
2414 | 	}
2415 | 
2416 | 	c := content(b, 10)
2417 | 	exp := c[:47]
2418 | 	if handle = realloc(f, h10, exp, false); handle != h10 {
2419 | 		t.Fatal(handle, h10)
2420 | 	}
2421 | 
2422 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2423 | 		t.Fatal(len(got), len(exp))
2424 | 	}
2425 | 
2426 | 	f = reaudit(t, f, name)
2427 | 
2428 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2429 | 		t.Fatal(len(got), len(exp))
2430 | 	}
2431 | 
2432 | 	used, total, err := f.audit() // c+4, c+4
2433 | 	if err != nil {
2434 | 		t.Fatal(err)
2435 | 	}
2436 | 
2437 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != -1 || free != 0 {
2438 | 		t.Fatal(difused, diftotal, free)
2439 | 	}
2440 | }
2441 | 
2442 | func TestRelocRealloc4Keep(t *testing.T) {
2443 | 	dir, name := temp()
2444 | 	defer os.RemoveAll(dir)
2445 | 
2446 | 	f, err := fcreate(name)
2447 | 	if err != nil {
2448 | 		t.Fatal(err)
2449 | 	}
2450 | 
2451 | 	defer func() {
2452 | 		if f != nil {
2453 | 			if err := f.Close(); err != nil {
2454 | 				t.Fatal(err)
2455 | 			}
2456 | 		}
2457 | 
2458 | 		f = nil
2459 | 		runtime.GC()
2460 | 		os.Remove(name)
2461 | 	}()
2462 | 
2463 | 	b := make([]byte, 61680)
2464 | 
2465 | 	h10 := alloc(f, nil)
2466 | 	_ = alloc(f, nil)
2467 | 	var handle int64
2468 | 	if handle = realloc(f, h10, b[:47], true); handle != h10 {
2469 | 		t.Fatal(handle, h10)
2470 | 	}
2471 | 
2472 | 	_ = alloc(f, nil)
2473 | 
2474 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2475 | 		t.Fatal(handle, h10)
2476 | 	}
2477 | 
2478 | 	used0, total0, err := f.audit() // c+4, c+5
2479 | 	if err != nil {
2480 | 		t.Fatal(err)
2481 | 	}
2482 | 
2483 | 	c := content(b, 10)
2484 | 	exp := c[:47]
2485 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
2486 | 		t.Fatal(handle, h10)
2487 | 	}
2488 | 
2489 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2490 | 		t.Fatal(len(got), len(exp))
2491 | 	}
2492 | 
2493 | 	f = reaudit(t, f, name)
2494 | 
2495 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2496 | 		t.Fatal(len(got), len(exp))
2497 | 	}
2498 | 
2499 | 	used, total, err := f.audit() // c+4, c+4
2500 | 	if err != nil {
2501 | 		t.Fatal(err)
2502 | 	}
2503 | 
2504 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != -1 || free != 0 {
2505 | 		t.Fatal(difused, diftotal, free)
2506 | 	}
2507 | }
2508 | 
2509 | func TestRelocRealloc5(t *testing.T) {
2510 | 	dir, name := temp()
2511 | 	defer os.RemoveAll(dir)
2512 | 
2513 | 	f, err := fcreate(name)
2514 | 	if err != nil {
2515 | 		t.Fatal(err)
2516 | 	}
2517 | 
2518 | 	defer func() {
2519 | 		if f != nil {
2520 | 			if err := f.Close(); err != nil {
2521 | 				t.Fatal(err)
2522 | 			}
2523 | 		}
2524 | 
2525 | 		f = nil
2526 | 		runtime.GC()
2527 | 		os.Remove(name)
2528 | 	}()
2529 | 
2530 | 	b := make([]byte, 61680)
2531 | 
2532 | 	h10 := alloc(f, nil)
2533 | 	_ = alloc(f, nil)
2534 | 	var handle int64
2535 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2536 | 		t.Fatal(handle, h10)
2537 | 	}
2538 | 
2539 | 	_ = alloc(f, nil)
2540 | 
2541 | 	used0, total0, err := f.audit() // c+4, c+4
2542 | 	if err != nil {
2543 | 		t.Fatal(err)
2544 | 	}
2545 | 
2546 | 	c := content(b, 10)
2547 | 	exp := c[:47]
2548 | 	if handle = realloc(f, h10, exp, false); handle != h10 {
2549 | 		t.Fatal(handle, h10)
2550 | 	}
2551 | 
2552 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2553 | 		t.Fatal(len(got), len(exp))
2554 | 	}
2555 | 
2556 | 	f = reaudit(t, f, name)
2557 | 
2558 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2559 | 		t.Fatal(len(got), len(exp))
2560 | 	}
2561 | 
2562 | 	used, total, err := f.audit() // c+4, c+5
2563 | 	if err != nil {
2564 | 		t.Fatal(err)
2565 | 	}
2566 | 
2567 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 {
2568 | 		t.Fatal(difused, diftotal, free)
2569 | 	}
2570 | }
2571 | 
2572 | func TestRelocRealloc5Keep(t *testing.T) {
2573 | 	dir, name := temp()
2574 | 	defer os.RemoveAll(dir)
2575 | 
2576 | 	f, err := fcreate(name)
2577 | 	if err != nil {
2578 | 		t.Fatal(err)
2579 | 	}
2580 | 
2581 | 	defer func() {
2582 | 		if f != nil {
2583 | 			if err := f.Close(); err != nil {
2584 | 				t.Fatal(err)
2585 | 			}
2586 | 		}
2587 | 
2588 | 		f = nil
2589 | 		runtime.GC()
2590 | 		os.Remove(name)
2591 | 	}()
2592 | 
2593 | 	b := make([]byte, 61680)
2594 | 
2595 | 	h10 := alloc(f, nil)
2596 | 	_ = alloc(f, nil)
2597 | 	var handle int64
2598 | 	if handle = realloc(f, h10, b[:31], true); handle != h10 {
2599 | 		t.Fatal(handle, h10)
2600 | 	}
2601 | 
2602 | 	_ = alloc(f, nil)
2603 | 
2604 | 	used0, total0, err := f.audit() // c+4, c+4
2605 | 	if err != nil {
2606 | 		t.Fatal(err)
2607 | 	}
2608 | 
2609 | 	c := content(b, 10)
2610 | 	exp := c[:47]
2611 | 	if handle = realloc(f, h10, exp, true); handle != h10 {
2612 | 		t.Fatal(handle, h10)
2613 | 	}
2614 | 
2615 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2616 | 		t.Fatal(len(got), len(exp))
2617 | 	}
2618 | 
2619 | 	f = reaudit(t, f, name)
2620 | 
2621 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2622 | 		t.Fatal(len(got), len(exp))
2623 | 	}
2624 | 
2625 | 	used, total, err := f.audit() // c+4, c+5
2626 | 	if err != nil {
2627 | 		t.Fatal(err)
2628 | 	}
2629 | 
2630 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 {
2631 | 		t.Fatal(difused, diftotal, free)
2632 | 	}
2633 | }
2634 | 
2635 | func TestRelocRealloc6(t *testing.T) {
2636 | 	dir, name := temp()
2637 | 	defer os.RemoveAll(dir)
2638 | 
2639 | 	f, err := fcreate(name)
2640 | 	if err != nil {
2641 | 		t.Fatal(err)
2642 | 	}
2643 | 
2644 | 	defer func() {
2645 | 		if f != nil {
2646 | 			if err := f.Close(); err != nil {
2647 | 				t.Fatal(err)
2648 | 			}
2649 | 		}
2650 | 
2651 | 		f = nil
2652 | 		runtime.GC()
2653 | 		os.Remove(name)
2654 | 	}()
2655 | 
2656 | 	b := make([]byte, 61680)
2657 | 
2658 | 	h10 := alloc(f, b[:31])
2659 | 	h20 := alloc(f, nil)
2660 | 	_ = alloc(f, nil)
2661 | 	free(f, h20)
2662 | 
2663 | 	used0, total0, err := f.audit() // c+2, c+3
2664 | 	if err != nil {
2665 | 		t.Fatal(err)
2666 | 	}
2667 | 
2668 | 	c := content(b, 10)
2669 | 	exp := c[:15]
2670 | 	if handle := realloc(f, h10, exp, false); handle != h10 {
2671 | 		t.Fatal(handle, h10)
2672 | 	}
2673 | 
2674 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2675 | 		t.Fatal(len(got), len(exp))
2676 | 	}
2677 | 
2678 | 	f = reaudit(t, f, name)
2679 | 
2680 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2681 | 		t.Fatal(len(got), len(exp))
2682 | 	}
2683 | 
2684 | 	used, total, err := f.audit() // c+2, c+3
2685 | 	if err != nil {
2686 | 		t.Fatal(err)
2687 | 	}
2688 | 
2689 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 1 {
2690 | 		t.Fatal(difused, diftotal, free)
2691 | 	}
2692 | }
2693 | 
2694 | func TestRelocRealloc6Keep(t *testing.T) {
2695 | 	dir, name := temp()
2696 | 	defer os.RemoveAll(dir)
2697 | 
2698 | 	f, err := fcreate(name)
2699 | 	if err != nil {
2700 | 		t.Fatal(err)
2701 | 	}
2702 | 
2703 | 	defer func() {
2704 | 		if f != nil {
2705 | 			if err := f.Close(); err != nil {
2706 | 				t.Fatal(err)
2707 | 			}
2708 | 		}
2709 | 
2710 | 		f = nil
2711 | 		runtime.GC()
2712 | 		os.Remove(name)
2713 | 	}()
2714 | 
2715 | 	b := make([]byte, 61680)
2716 | 
2717 | 	h10 := alloc(f, b[:31])
2718 | 	h20 := alloc(f, nil)
2719 | 	_ = alloc(f, nil)
2720 | 	free(f, h20)
2721 | 
2722 | 	used0, total0, err := f.audit() // c+2, c+3
2723 | 	if err != nil {
2724 | 		t.Fatal(err)
2725 | 	}
2726 | 
2727 | 	c := content(b, 10)
2728 | 	exp := c[:15]
2729 | 	if handle := realloc(f, h10, exp, true); handle != h10 {
2730 | 		t.Fatal(handle, h10)
2731 | 	}
2732 | 
2733 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2734 | 		t.Fatal(len(got), len(exp))
2735 | 	}
2736 | 
2737 | 	f = reaudit(t, f, name)
2738 | 
2739 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) {
2740 | 		t.Fatal(len(got), len(exp))
2741 | 	}
2742 | 
2743 | 	used, total, err := f.audit() // c+2, c+3
2744 | 	if err != nil {
2745 | 		t.Fatal(err)
2746 | 	}
2747 | 
2748 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 1 {
2749 | 		t.Fatal(difused, diftotal, free)
2750 | 	}
2751 | }
2752 | 
2753 | func TestFreespaceReuse(t *testing.T) {
2754 | 	dir, name := temp()
2755 | 	defer os.RemoveAll(dir)
2756 | 
2757 | 	f, err := fcreate(name)
2758 | 	if err != nil {
2759 | 		t.Fatal(err)
2760 | 	}
2761 | 
2762 | 	defer func() {
2763 | 		if f != nil {
2764 | 			if err := f.Close(); err != nil {
2765 | 				t.Fatal(err)
2766 | 			}
2767 | 		}
2768 | 
2769 | 		f = nil
2770 | 		runtime.GC()
2771 | 		os.Remove(name)
2772 | 	}()
2773 | 
2774 | 	b := make([]byte, 61680)
2775 | 	c := content(b, 10)
2776 | 
2777 | 	c10 := c[0 : 0+15]
2778 | 	c20 := c[16:63]
2779 | 	c50 := c[64 : 64+15]
2780 | 	h10 := alloc(f, c10)
2781 | 	h201 := alloc(f, nil)
2782 | 	h202 := alloc(f, nil)
2783 | 	h203 := alloc(f, nil)
2784 | 	h50 := alloc(f, c50)
2785 | 	free(f, h201)
2786 | 	free(f, h202)
2787 | 	free(f, h203)
2788 | 	used0, total0, err := f.audit() // c+2, c+3
2789 | 	if err != nil {
2790 | 		t.Fatal(err)
2791 | 	}
2792 | 
2793 | 	h20 := alloc(f, c20)
2794 | 
2795 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) {
2796 | 		t.Fatal()
2797 | 	}
2798 | 
2799 | 	if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) {
2800 | 		t.Fatal()
2801 | 	}
2802 | 
2803 | 	if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) {
2804 | 		t.Fatal()
2805 | 	}
2806 | 
2807 | 	f = reaudit(t, f, name)
2808 | 
2809 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) {
2810 | 		t.Fatal()
2811 | 	}
2812 | 
2813 | 	if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) {
2814 | 		t.Fatal()
2815 | 	}
2816 | 
2817 | 	if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) {
2818 | 		t.Fatal()
2819 | 	}
2820 | 
2821 | 	used, total, err := f.audit() // c+3, c+3
2822 | 	if err != nil {
2823 | 		t.Fatal(err)
2824 | 	}
2825 | 
2826 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 0 || free != 0 {
2827 | 		t.Fatal(difused, diftotal, free)
2828 | 	}
2829 | }
2830 | 
2831 | func TestFreespaceReuse2(t *testing.T) {
2832 | 	dir, name := temp()
2833 | 	defer os.RemoveAll(dir)
2834 | 
2835 | 	f, err := fcreate(name)
2836 | 	if err != nil {
2837 | 		t.Fatal(err)
2838 | 	}
2839 | 
2840 | 	defer func() {
2841 | 		if f != nil {
2842 | 			if err := f.Close(); err != nil {
2843 | 				t.Fatal(err)
2844 | 			}
2845 | 		}
2846 | 
2847 | 		f = nil
2848 | 		runtime.GC()
2849 | 		os.Remove(name)
2850 | 	}()
2851 | 
2852 | 	b := make([]byte, 61680)
2853 | 	c := content(b, 10)
2854 | 
2855 | 	c10 := c[0 : 0+15]
2856 | 	c20 := c[16:47]
2857 | 	c50 := c[64 : 64+15]
2858 | 	h10 := alloc(f, c10)
2859 | 	h201 := alloc(f, nil)
2860 | 	h202 := alloc(f, nil)
2861 | 	h203 := alloc(f, nil)
2862 | 	h50 := alloc(f, c50)
2863 | 	free(f, h201)
2864 | 	free(f, h202)
2865 | 	free(f, h203)
2866 | 	used0, total0, err := f.audit() // c+2, c+3
2867 | 	if err != nil {
2868 | 		t.Fatal(err)
2869 | 	}
2870 | 
2871 | 	h20 := alloc(f, c20)
2872 | 
2873 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) {
2874 | 		t.Fatal()
2875 | 	}
2876 | 
2877 | 	if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) {
2878 | 		t.Fatal()
2879 | 	}
2880 | 
2881 | 	if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) {
2882 | 		t.Fatal()
2883 | 	}
2884 | 
2885 | 	f = reaudit(t, f, name)
2886 | 
2887 | 	if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) {
2888 | 		t.Fatal()
2889 | 	}
2890 | 
2891 | 	if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) {
2892 | 		t.Fatal()
2893 | 	}
2894 | 
2895 | 	if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) {
2896 | 		t.Fatal()
2897 | 	}
2898 | 
2899 | 	used, total, err := f.audit() // c+3, c+4
2900 | 	if err != nil {
2901 | 		t.Fatal(err)
2902 | 	}
2903 | 
2904 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 1 || free != 1 {
2905 | 		t.Fatal(difused, diftotal, free)
2906 | 	}
2907 | }
2908 | 
2909 | func testBug1(t *testing.T, swap bool) {
2910 | 	// Free lists table item for size 3856 points to list of free blocks
2911 | 	// NOT of size 3856 but at least 3856.
2912 | 
2913 | 	dir, name := temp()
2914 | 	defer os.RemoveAll(dir)
2915 | 
2916 | 	f, err := fcreate(name)
2917 | 	if err != nil {
2918 | 		t.Fatal(err)
2919 | 	}
2920 | 
2921 | 	defer func() {
2922 | 		if f != nil {
2923 | 			if err := f.Close(); err != nil {
2924 | 				t.Fatal(err)
2925 | 			}
2926 | 		}
2927 | 
2928 | 		f = nil
2929 | 		runtime.GC()
2930 | 		os.Remove(name)
2931 | 	}()
2932 | 
2933 | 	_ = alloc(f, nil)
2934 | 	b := make([]byte, 61680)
2935 | 	f1 := alloc(f, b)
2936 | 	f2 := alloc(f, b)
2937 | 	_ = alloc(f, nil)
2938 | 
2939 | 	used0, total0, err := f.audit() // c+4, c+4
2940 | 	if err != nil {
2941 | 		t.Fatal(err)
2942 | 	}
2943 | 
2944 | 	if swap {
2945 | 		f1, f2 = f2, f1
2946 | 	}
2947 | 	free(f, f1)
2948 | 	free(f, f2)
2949 | 	_ = alloc(f, nil)
2950 | 
2951 | 	f = reaudit(t, f, name)
2952 | 
2953 | 	used, total, err := f.audit() // c+3, c+4
2954 | 	if err != nil {
2955 | 		t.Fatal(err)
2956 | 	}
2957 | 
2958 | 	if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != 0 || free != 1 {
2959 | 		t.Fatal(difused, diftotal, free)
2960 | 	}
2961 | }
2962 | 
2963 | func TestBug1(t *testing.T) {
2964 | 	testBug1(t, false)
2965 | 	testBug1(t, true)
2966 | }
2967 | 
2968 | func TestMix(t *testing.T) {
2969 | 	if testing.Short() {
2970 | 		t.Log("skipped")
2971 | 		return
2972 | 	}
2973 | 
2974 | 	const (
2975 | 		n = 1 << 10
2976 | 	)
2977 | 
2978 | 	if testing.Short() {
2979 | 		t.Log("skipped")
2980 | 		return
2981 | 	}
2982 | 
2983 | 	t.Log(n)
2984 | 	dir, name := temp()
2985 | 	defer os.RemoveAll(dir)
2986 | 
2987 | 	f, err := fcreate(name)
2988 | 	if err != nil {
2989 | 		t.Fatal(err)
2990 | 	}
2991 | 
2992 | 	defer func() {
2993 | 		if f != nil {
2994 | 			if err := f.Close(); err != nil {
2995 | 				t.Fatal(err)
2996 | 			}
2997 | 		}
2998 | 
2999 | 		f = nil
3000 | 		runtime.GC()
3001 | 		os.Remove(name)
3002 | 	}()
3003 | 
3004 | 	b := make([]byte, 61680)
3005 | 	rng, err := mathutil.NewFC32(0, n-1, true)
3006 | 	if err != nil {
3007 | 		t.Fatal(err)
3008 | 	}
3009 | 
3010 | 	ha := make([]int64, n)
3011 | 	payload := 0
3012 | 
3013 | 	t0 := time.Now()
3014 | 	// Alloc n block with upper half of content
3015 | 	for _ = range ha {
3016 | 		r := rng.Next()
3017 | 		c := content(b, int64(r))
3018 | 		c = c[len(c)/2:]
3019 | 		ha[r] = alloc(f, c)
3020 | 		payload += len(c)
3021 | 	}
3022 | 	dt := float64(time.Now().Sub(t0)) / 1e9
3023 | 	t.Logf("write time A %.3g", dt)
3024 | 
3025 | 	// verify
3026 | 	f = reaudit(t, f, name)
3027 | 	t.Logf("size A %d for %d bytes (fill factor %3.1f%%)", f.atoms<<4, payload, 100*float64(payload)/float64(f.atoms<<4))
3028 | 	t0 = time.Now()
3029 | 	for _ = range ha {
3030 | 		r := rng.Next()
3031 | 		c := content(b, int64(r))
3032 | 		c = c[len(c)/2:]
3033 | 		if got, _ := f.readUsed(ha[r]); !bytes.Equal(got, c) {
3034 | 			t.Fatal()
3035 | 		}
3036 | 	}
3037 | 	dt = float64(time.Now().Sub(t0)) / 1e9
3038 | 	t.Logf("read time A %.3g", dt)
3039 | 	// free half of the blocks
3040 | 	t0 = time.Now()
3041 | 	for i := 0; i < n/2; i++ {
3042 | 		free(f, ha[i])
3043 | 		ha[i] = 0
3044 | 	}
3045 | 	dt = float64(time.Now().Sub(t0)) / 1e9
3046 | 	t.Logf("free time A %.3g", dt)
3047 | 
3048 | 	// verify
3049 | 	f = reaudit(t, f, name)
3050 | 	t.Logf("size B %d (freeing half of the blocks)", f.atoms<<4)
3051 | 	t0 = time.Now()
3052 | 	for _ = range ha {
3053 | 		r := rng.Next()
3054 | 		h := ha[r]
3055 | 		if h == 0 {
3056 | 			continue
3057 | 		}
3058 | 
3059 | 		c := content(b, int64(r))
3060 | 		c = c[len(c)/2:]
3061 | 		if got, _ := f.readUsed(h); !bytes.Equal(got, c) {
3062 | 			t.Fatal()
3063 | 		}
3064 | 	}
3065 | 	dt = float64(time.Now().Sub(t0)) / 1e9
3066 | 	t.Logf("read time B %.3g", dt)
3067 | 
3068 | 	// reloc extend
3069 | 	t0 = time.Now()
3070 | 	for _ = range ha {
3071 | 		r := rng.Next()
3072 | 		h := ha[r]
3073 | 		if h == 0 {
3074 | 			continue
3075 | 		}
3076 | 
3077 | 		c := content(b, int64(r))
3078 | 		//f = reaudit(t, f, name)
3079 | 		if h2 := realloc(f, h, c, true); h2 != h {
3080 | 			t.Fatal()
3081 | 		}
3082 | 	}
3083 | 	dt = float64(time.Now().Sub(t0)) / 1e9
3084 | 	t.Logf("realoc time B %.3g", dt)
3085 | 
3086 | 	// verify
3087 | 	f = reaudit(t, f, name)
3088 | 	t.Logf("size C %d for %d bytes (reallocated all used blocks to double size, fill factor %3.1f%%", f.atoms<<4, payload, 100*float64(payload)/float64(f.atoms<<4))
3089 | 
3090 | 	t0 = time.Now()
3091 | 	for _ = range ha {
3092 | 		r := rng.Next()
3093 | 		h := ha[r]
3094 | 		if h == 0 {
3095 | 			continue
3096 | 		}
3097 | 
3098 | 		c := content(b, int64(r))
3099 | 		if got, _ := f.readUsed(ha[r]); !bytes.Equal(got, c) {
3100 | 			t.Fatal()
3101 | 		}
3102 | 	}
3103 | 	dt = float64(time.Now().Sub(t0)) / 1e9
3104 | 	t.Logf("read time C %.3g", dt)
3105 | }
3106 | 


--------------------------------------------------------------------------------
/falloc/docs.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // blame: jnml, labs.nic.cz
  6 | 
  7 | /*
  8 | 
  9 | WIP: Package falloc provides allocation/deallocation of space within a
 10 | file/store (WIP, unstable API).
 11 | 
 12 | Overall structure:
 13 |  File == n blocks.
 14 |  Block == n atoms.
 15 |  Atom == 16 bytes.
 16 | 
 17 | x6..x0 == least significant 7 bytes of a 64 bit integer, highest (7th) byte is
 18 | 0 and is not stored in the file.
 19 | 
 20 | Block first byte
 21 | 
 22 | Aka block type tag.
 23 | 
 24 | ------------------------------------------------------------------------------
 25 | 
 26 | 0xFF: Free atom (free block of size 1).
 27 |  +------++---------++---------++------+
 28 |  |  0   ||  1...7  || 8...14  ||  15  |
 29 |  +------++---------++---------++------+
 30 |  | 0xFF || p6...p0 || n6...n0 || 0xFF |
 31 |  +------++---------++---------++------+
 32 | 
 33 | Link to the previous free block (atom addressed) is p6...p0, next dtto in
 34 | n6...n0.  Doubly linked lists of "compatible" free blocks allows for free space
 35 | reclaiming and merging.  "Compatible" == of size at least some K. Heads of all
 36 | such lists are organized per K or intervals of Ks elsewhere.
 37 | 
 38 | ------------------------------------------------------------------------------
 39 | 
 40 | 0xFE: Free block, size == s6...s0 atoms.
 41 |  +------++---------++---------++---------++--
 42 |  |  +0  ||  1...7  || 8...14  || 15...21 || 22...16*size-1
 43 |  +------++---------++---------++---------++--
 44 |  | 0xFE || p6...p0 || n6...n0 || s6...s0 || ...
 45 |  +------++---------++---------++---------++--
 46 | 
 47 | Prev and next links as in the 0xFF first byte case.  End of this block - see
 48 | "Block last byte": 0xFE bellow. Data between == undefined.
 49 | 
 50 | ------------------------------------------------------------------------------
 51 | 
 52 | 0xFD:  Relocated block.
 53 |  +------++---------++-----------++------+
 54 |  |  0   ||  1...7  ||  8...14   ||  15  |
 55 |  +------++---------++-----------++------+
 56 |  | 0xFD || r6...r0 || undefined || 0x00 | // == used block
 57 |  +------++---------++-----------++------+
 58 | 
 59 | Relocation link is r6..r0 == atom address. Relocations MUST NOT chain and MUST
 60 | point to a "content" block, i.e. one with the first byte in 0x00...0xFC.
 61 | 
 62 | Relocated block allows to permanently assign a handle/file pointer ("atom"
 63 | address) to some content and resize the content anytime afterwards w/o having
 64 | to update all the possible existing references to the original handle.
 65 | 
 66 | ------------------------------------------------------------------------------
 67 | 
 68 | 0xFC: Used long block.
 69 |  +------++---------++--------------------++---------+---+
 70 |  |  0   ||  1...2  ||      3...N+2       ||         |   |
 71 |  +------++---------++--------------------++---------+---+
 72 |  | 0xFC || n1...n0 || N bytes of content || padding | Z |
 73 |  +------++---------++--------------------++---------+---+
 74 | 
 75 | This block type is used for content of length in N == 238...61680 bytes. N is
 76 | encoded as a 2 byte unsigned integer n1..n0 in network byte order. Values
 77 | bellow 238 are reserved, those content lengths are to be carried by the
 78 | 0x00..0xFB block types.
 79 | 
 80 |  1. n in 0x00EE...0xF0F0 is used for content under the same rules
 81 |     as in the 0x01..0xED type.
 82 | 
 83 |  2. If the last byte of the content is not the last byte of an atom then
 84 |     the last byte of the block is 0x00.
 85 | 
 86 |  3. If the last byte of the content IS the last byte of an atom:
 87 | 
 88 |    3.1 If the last byte of content is in 0x00..0xFD then everything is OK.
 89 | 
 90 |    3.2 If the last byte of content is 0xFE or 0xFF then the escape
 91 |        via n > 0xF0F0 MUST be used AND the block's last byte is 0x00 or 0x01,
 92 |        meaning value 0xFE and 0xFF respectively.
 93 | 
 94 |  4. n in 0xF0F1...0xFFFF is like the escaped 0xEE..0xFB block.
 95 |     N == 13 + 16(n - 0xF0F1).
 96 | 
 97 | Discussion of the padding and Z fields - see the 0x01..0xED block type.
 98 | 
 99 | ------------------------------------------------------------------------------
100 | 
101 | 0xEE...0xFB: Used escaped short block.
102 |  +---++----------------------++---+
103 |  | 0 ||        1...N-1       ||   |
104 |  +---++----------------------++---+
105 |  | X || N-1 bytes of content || Z |
106 |  +---++----------------------++---+
107 | 
108 | N == 15 + 16(X - 0xEE). Z is the content last byte encoded as follows.
109 | 
110 | case Z == 0x00:	The last byte of content is 0xFE
111 | 
112 | case Z == 0x01:	The last byte of content is 0xFF
113 | 
114 | ------------------------------------------------------------------------------
115 | 
116 | 0x01...0xED: Used short block.
117 |  +---++--------------------++---------+---+
118 |  | 0 ||        1...N       ||         |   |
119 |  +---++--------------------++---------+---+
120 |  | N || N bytes of content || padding | Z |
121 |  +---++--------------------++---------+---+
122 | 
123 | This block type is used for content of length in 1...237 bytes.  The value of
124 | the "padding" field, if of non zero length, is undefined.
125 | 
126 | If the last byte of content is the last byte of an atom (== its file byte
127 | offset & 0xF == 0xF) then such last byte MUST be in 0x00...0xFD.
128 | 
129 | If the last byte of content is the last byte of an atom AND the last byte of
130 | content is 0xFE or 0xFF then the short escape block type (0xEE...0xFB) MUST be
131 | used.
132 | 
133 | If the last byte of content is not the last byte of an atom, then the last byte
134 | of such block, i.e. the Z field, which is also a last byte of some atom, MUST
135 | be 0x00 (i.e. the used block marker).  Other "tail" values are reserved.
136 | 
137 | ------------------------------------------------------------------------------
138 | 
139 | 0x00: Used empty block.
140 |  +------++-----------++------+
141 |  |  0   ||  1...14   ||  15  |
142 |  +------++-----------++------+
143 |  | 0x00 || undefined || 0x00 | // == used block, other "tail" values reserved.
144 |  +------++-----------++------+
145 | 
146 | All of the rules for 0x01..0xED applies. Depicted only for its different
147 | semantics (e.g. an allocated [existing] string but with length of zero).
148 | 
149 | ==============================================================================
150 | 
151 | Block last byte
152 | 
153 | ------------------------------------------------------------------------------
154 | 
155 | 0xFF: Free atom. Layout - see "Block first byte": FF.
156 | 
157 | ------------------------------------------------------------------------------
158 | 
159 | 0xFE: Free block, size n atoms. Preceding 7 bytes == size (s6...s0) of the free
160 | block in atoms, network byte order
161 |    --++---------++------+
162 |      || -8...-2 ||  -1  |
163 |    --++---------++------+
164 |  ... || s6...s0 || 0xFE | <- block's last byte
165 |    --++---------++------+
166 | 
167 | Layout at start of this block - see "Block first byte": FE.
168 | 
169 | ------------------------------------------------------------------------------
170 | 
171 | 0x00...0xFD: Used (non free) block.
172 | 
173 | ==============================================================================
174 | 
175 | Free lists table
176 | 
177 | The free lists table content is stored in the standard layout of a used block.
178 | 
179 | A table item is a 7 byte size field followed by a 7 byte atom address field
180 | (both in network byte order), thus every item is 14 contiguous bytes. The
181 | item's address field is pointing to a free block.  The size field determines
182 | the minimal size (in atoms) of free blocks on that list.
183 | 
184 | The free list table is n above items, thus the content has 14n bytes. Note that
185 | the largest block content is 61680 bytes and as there are 14 bytes per table
186 | item, so the table is limited to at most 4405 entries.
187 | 
188 | Items in the table do not have to be sorted according to their size field values.
189 | 
190 | No two items can have the same value of the size field.
191 | 
192 | When freeing blocks, the block MUST be linked into an item list with the
193 | highest possible size field, which is less or equal to the number of atoms in
194 | the new free block.
195 | 
196 | When freeing a block, the block MUST be first merged with any adjacent free
197 | blocks (thus possibly creating a bigger free block) using information derived
198 | from the adjacent blocks first and last bytes. Such merged free blocks MUST be
199 | removed from their original doubly linked lists. Afterwards the new bigger free
200 | block is put to the free list table in the appropriate item.
201 | 
202 | Items with address field == 0 are legal. Such item is a placeholder for a empty
203 | list of free blocks of the item's size.
204 | 
205 | Items with size field == 0 are legal. Such item is a placeholder, used e.g. to
206 | avoid further reallocations/redirecting of the free lists table.
207 | 
208 | The largest possible allocation request (for content length 61680 bytes) is
209 | 0xF10 (3856) atoms.  All free blocks of this or bigger size are presumably put
210 | into a single table item with the size 3856. It may be useful to additionally
211 | have a free lists table item which links free blocks of some bigger size (say
212 | 1M+) and then use the OS sparse file support (if present) to save the physical
213 | space used by such free blocks.
214 | 
215 | Smaller (<3856 atoms) free blocks can be organized exactly (every distinct size
216 | has its table item) or the sizes can run using other schema like e.g. "1, 2,
217 | 4, 8, ..." (powers of 2) or "1, 2, 3, 5, 8, 13, ..." (the Fibonacci sequence)
218 | or they may be fine tuned to a specific usage pattern.
219 | 
220 | ==============================================================================
221 | 
222 | Header
223 | 
224 | The first block of a file (atom address == file offset == 0) is the file header.
225 | The header block has the standard layout of a used short non escaped block.
226 | 
227 | Special conditions apply: The header block and its content MUST be like this:
228 | 
229 |  +------+---------+---------+------+
230 |  |  0   |  1...7  | 8...14  |  15  |
231 |  +------+---------+---------+------+
232 |  | 0x0F | m6...m0 | f6...f0 | FLTT |
233 |  +------+---------+---------+------+
234 | 
235 | m6..m0 is a "magic" value 0xF1C1A1FE51B1E.
236 | 
237 | f6...f0 is the atom address of the free lists table (discussed elsewhere).
238 | If f6...f0 == 0x00 the there is no free lists table (yet).
239 | 
240 | FLTT describes the type of the Free List Table. Currently defined values:
241 | 
242 | ------------------------------------------------------------------------------
243 | 
244 | FLTT == 0: Free List Table is fixed at atom address 2. It has a fixed size for 3856 entries
245 | for free list of size 1..3855 atoms and the last is for the list of free block >= 3856 atoms.
246 | */
247 | package falloc
248 | 
249 | const (
250 | 	INVALID_HANDLE = Handle(-1)
251 | )
252 | 


--------------------------------------------------------------------------------
/falloc/error.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // blame: jnml, labs.nic.cz
  6 | 
  7 | package falloc
  8 | 
  9 | import "fmt"
 10 | 
 11 | // EBadRequest is an error produced for invalid operation, e.g. for data of more than maximum allowed.
 12 | type EBadRequest struct {
 13 | 	Name string
 14 | 	Size int
 15 | }
 16 | 
 17 | func (e *EBadRequest) Error() string {
 18 | 	return fmt.Sprintf("%s: size %d", e.Name, e.Size)
 19 | }
 20 | 
 21 | // EClose is a file/store close error.
 22 | type EClose struct {
 23 | 	Name string
 24 | 	Err  error
 25 | }
 26 | 
 27 | func (e *EClose) Error() string {
 28 | 	return fmt.Sprintf("%sx: %s", e.Name, e.Err)
 29 | }
 30 | 
 31 | // ECorrupted is a file/store format error.
 32 | type ECorrupted struct {
 33 | 	Name string
 34 | 	Ofs  int64
 35 | }
 36 | 
 37 | func (e *ECorrupted) Error() string {
 38 | 	return fmt.Sprintf("%s: corrupted data @%#x", e.Name, e.Ofs)
 39 | }
 40 | 
 41 | // ECreate is a file/store create error.
 42 | type ECreate struct {
 43 | 	Name string
 44 | 	Err  error
 45 | }
 46 | 
 47 | func (e *ECreate) Error() string {
 48 | 	return fmt.Sprintf("%s: %s", e.Name, e.Err)
 49 | }
 50 | 
 51 | // EFreeList is a file/store format error.
 52 | type EFreeList struct {
 53 | 	Name  string
 54 | 	Size  int64
 55 | 	Block int64
 56 | }
 57 | 
 58 | func (e *EFreeList) Error() string {
 59 | 	return fmt.Sprintf("%s: invalid free list item, size %#x, block %#x", e.Name, e.Size, e.Block)
 60 | }
 61 | 
 62 | // EHandle is an error type reported for invalid Handles.
 63 | type EHandle struct {
 64 | 	Name   string
 65 | 	Handle Handle
 66 | }
 67 | 
 68 | func (e EHandle) Error() string {
 69 | 	return fmt.Sprintf("%s: invalid handle %#x", e.Name, e.Handle)
 70 | }
 71 | 
 72 | // EHeader is a file/store format error.
 73 | type EHeader struct {
 74 | 	Name     string
 75 | 	Header   []byte
 76 | 	Expected []byte
 77 | }
 78 | 
 79 | func (e *EHeader) Error() string {
 80 | 	return fmt.Sprintf("%s: invalid header, got [% x], expected [% x]", e.Name, e.Header, e.Expected)
 81 | }
 82 | 
 83 | // ENullHandle is a file/store access error via a null handle.
 84 | type ENullHandle string
 85 | 
 86 | func (e ENullHandle) Error() string {
 87 | 	return fmt.Sprintf("%s: access via null handle", e)
 88 | }
 89 | 
 90 | // EOpen is a file/store open error.
 91 | type EOpen struct {
 92 | 	Name string
 93 | 	Err  error
 94 | }
 95 | 
 96 | func (e *EOpen) Error() string {
 97 | 	return fmt.Sprintf("%s: %s", e.Name, e.Err)
 98 | }
 99 | 
100 | // ERead is a file/store read error.
101 | type ERead struct {
102 | 	Name string
103 | 	Ofs  int64
104 | 	Err  error
105 | }
106 | 
107 | func (e *ERead) Error() string {
108 | 	return fmt.Sprintf("%s, %#x: %s", e.Name, e.Ofs, e.Err)
109 | }
110 | 
111 | // ESize is a file/store size error.
112 | type ESize struct {
113 | 	Name string
114 | 	Size int64
115 | }
116 | 
117 | func (e *ESize) Error() string {
118 | 	return fmt.Sprintf("%s: invalid size %#x(%d), size %%16 != 0", e.Name, e.Size, e.Size)
119 | }
120 | 
121 | // EWrite is a file/store write error.
122 | type EWrite struct {
123 | 	Name string
124 | 	Ofs  int64
125 | 	Err  error
126 | }
127 | 
128 | func (e *EWrite) Error() string {
129 | 	return fmt.Sprintf("%s, %#x: %s", e.Name, e.Ofs, e.Err)
130 | }
131 | 


--------------------------------------------------------------------------------
/falloc/falloc.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // blame: jnml, labs.nic.cz
  6 | 
  7 | /*
  8 | 
  9 | This is an mostly (WIP) conforming implementation of the "specs" in docs.go.
 10 | 
 11 | The main incompletness is support for only one kind of FTL, though this table kind is still per "specs".
 12 | 
 13 | */
 14 | 
 15 | package falloc
 16 | 
 17 | import (
 18 | 	"bytes"
 19 | 	"github.com/cznic/fileutil/storage"
 20 | 	"sync"
 21 | )
 22 | 
 23 | // Handle is a reference to a block in a file/store.
 24 | // Handle is an uint56 wrapped in an in64, i.e. the most significant byte must be always zero.
 25 | type Handle int64
 26 | 
 27 | // Put puts the 7 least significant bytes of h into b. The MSB of h should be zero.
 28 | func (h Handle) Put(b []byte) {
 29 | 	for ofs := 6; ofs >= 0; ofs-- {
 30 | 		b[ofs] = byte(h)
 31 | 		h >>= 8
 32 | 	}
 33 | }
 34 | 
 35 | // Get gets the 7 least significant bytes of h from b. The MSB of h is zeroed.
 36 | func (h *Handle) Get(b []byte) {
 37 | 	var x Handle
 38 | 	for ofs := 0; ofs <= 6; ofs++ {
 39 | 		x = x<<8 | Handle(b[ofs])
 40 | 	}
 41 | 	*h = x
 42 | }
 43 | 
 44 | // File is a file/store with space allocation/deallocation support.
 45 | type File struct {
 46 | 	f       storage.Accessor
 47 | 	atoms   int64       // current file size in atom units
 48 | 	canfree int64       // only blocks >= canfree can be subject to Free()
 49 | 	freetab [3857]int64 // freetab[0] is unused, freetab[1] is size 1 ptr, freetab[2] is size 2 ptr, ...
 50 | 	rwm     sync.RWMutex
 51 | }
 52 | 
 53 | func (f *File) read(b []byte, off int64) {
 54 | 	if n, err := f.f.ReadAt(b, off); n != len(b) {
 55 | 		panic(&ERead{f.f.Name(), off, err})
 56 | 	}
 57 | }
 58 | 
 59 | func (f *File) write(b []byte, off int64) {
 60 | 	if n, err := f.f.WriteAt(b, off); n != len(b) {
 61 | 		panic(&EWrite{f.f.Name(), off, err})
 62 | 	}
 63 | }
 64 | 
 65 | var ( // R/O
 66 | 	hdr   = []byte{0x0f, 0xf1, 0xc1, 0xa1, 0xfe, 0xa5, 0x1b, 0x1e, 0, 0, 0, 0, 0, 0, 2, 0} // free lists table @2
 67 | 	empty = make([]byte, 16)
 68 | 	zero  = []byte{0}
 69 | 	zero7 = make([]byte, 7)
 70 | )
 71 | 
 72 | // New returns a new File backed by store or an error if any.
 73 | // Any existing data in store are discarded.
 74 | func New(store storage.Accessor) (f *File, err error) {
 75 | 	f = &File{f: store}
 76 | 	return f, storage.Mutate(store, func() (err error) {
 77 | 		if err = f.f.Truncate(0); err != nil {
 78 | 			return &ECreate{f.f.Name(), err}
 79 | 		}
 80 | 
 81 | 		if _, err = f.Alloc(hdr[1:]); err != nil { //TODO internal panicking versions of the exported fns.
 82 | 			return
 83 | 		}
 84 | 
 85 | 		if _, err = f.Alloc(nil); err != nil { // (empty) root @1
 86 | 			return
 87 | 		}
 88 | 
 89 | 		b := make([]byte, 3856*14)
 90 | 		for i := 1; i <= 3856; i++ {
 91 | 			Handle(i).Put(b[(i-1)*14:])
 92 | 		}
 93 | 		if _, err = f.Alloc(b); err != nil {
 94 | 			return
 95 | 		}
 96 | 
 97 | 		f.canfree = f.atoms
 98 | 		return
 99 | 	})
100 | }
101 | 
102 | // Open returns a new File backed by store or an error if any.
103 | // Store already has to be in a valid format.
104 | func Open(store storage.Accessor) (f *File, err error) {
105 | 	defer func() {
106 | 		if e := recover(); e != nil {
107 | 			f = nil
108 | 			err = e.(error)
109 | 		}
110 | 	}()
111 | 
112 | 	fi, err := store.Stat()
113 | 	if err != nil {
114 | 		panic(&EOpen{store.Name(), err})
115 | 	}
116 | 
117 | 	fs := fi.Size()
118 | 	if fs&0xf != 0 {
119 | 		panic(&ESize{store.Name(), fi.Size()})
120 | 	}
121 | 
122 | 	f = &File{f: store, atoms: fs >> 4}
123 | 	b := make([]byte, len(hdr))
124 | 	f.read(b, 0)
125 | 	if !bytes.Equal(b, hdr) {
126 | 		panic(&EHeader{store.Name(), b, append([]byte{}, hdr...)})
127 | 	}
128 | 
129 | 	var atoms int64
130 | 	b, atoms = f.readUsed(2)
131 | 	f.canfree = atoms + 2
132 | 	ofs := 0
133 | 	var size, p Handle
134 | 	for ofs < len(b) {
135 | 		size.Get(b[ofs:])
136 | 		ofs += 7
137 | 		p.Get(b[ofs:])
138 | 		ofs += 7
139 | 		if sz, pp := int64(size), int64(p); size == 0 || size > 3856 || (pp != 0 && pp < f.canfree) || pp<<4 > fs-16 {
140 | 			panic(&EFreeList{store.Name(), sz, pp})
141 | 		}
142 | 
143 | 		f.freetab[size] = int64(p)
144 | 	}
145 | 	return
146 | }
147 | 
148 | // Accessor returns the File's underlying Accessor.
149 | func (f *File) Accessor() storage.Accessor {
150 | 	return f.f
151 | }
152 | 
153 | // Close closes f and returns an error if any.
154 | func (f *File) Close() (err error) {
155 | 	return storage.Mutate(f.Accessor(), func() (err error) {
156 | 		if err = f.f.Close(); err != nil {
157 | 			err = &EClose{f.f.Name(), err}
158 | 		}
159 | 		return
160 | 	})
161 | }
162 | 
163 | // Root returns the handle of the DB root (top level directory, ...).
164 | func (f *File) Root() Handle {
165 | 	return 1
166 | }
167 | 
168 | func (f *File) readUsed(atom int64) (content []byte, atoms int64) {
169 | 	b, redirected := make([]byte, 7), false
170 | redir:
171 | 	ofs := atom << 4
172 | 	f.read(b[:1], ofs)
173 | 	switch pre := b[0]; {
174 | 	default:
175 | 		panic(&ECorrupted{f.f.Name(), ofs})
176 | 	case pre == 0x00: // Empty block
177 | 	case pre >= 1 && pre <= 237: // Short
178 | 		content = make([]byte, pre)
179 | 		f.read(content, ofs+1)
180 | 	case pre >= 0xee && pre <= 0xfb: // Short esc
181 | 		content = make([]byte, 15+16*(pre-0xee))
182 | 		f.read(content, ofs+1)
183 | 		content[len(content)-1] += 0xfe
184 | 	case pre == 0xfc: // Long
185 | 		f.read(b[:2], ofs+1)
186 | 		n := int(b[0])<<8 + int(b[1])
187 | 		switch {
188 | 		default:
189 | 			panic(&ECorrupted{f.f.Name(), ofs + 1})
190 | 		case n >= 238 && n <= 61680: // Long non esc
191 | 			content = make([]byte, n)
192 | 			f.read(content, ofs+3)
193 | 		case n >= 61681: // Long esc
194 | 			content = make([]byte, 13+16*(n-0xf0f1))
195 | 			f.read(content, ofs+3)
196 | 			content[len(content)-1] += 0xfe
197 | 		}
198 | 	case pre == 0xfd: // redir
199 | 		if redirected {
200 | 			panic(&ECorrupted{f.f.Name(), ofs})
201 | 		}
202 | 
203 | 		f.read(b[:7], ofs+1)
204 | 		(*Handle)(&atom).Get(b)
205 | 		redirected = true
206 | 		goto redir
207 | 	}
208 | 	return content, rq2Atoms(len(content))
209 | }
210 | 
211 | func (f *File) writeUsed(b []byte, atom int64) {
212 | 	n := len(b)
213 | 	switch ofs, atoms, endmark := atom<<4, rq2Atoms(n), true; {
214 | 	default:
215 | 		panic("internal error")
216 | 	case n == 0:
217 | 		f.write(empty, ofs)
218 | 	case n <= 237:
219 | 		if (n+1)&0xf == 0 { // content end == atom end
220 | 			if v := b[n-1]; v >= 0xfe { // escape
221 | 				pre := []byte{byte((16*0xee + n - 15) >> 4)}
222 | 				f.write(pre, ofs)
223 | 				f.write(b[:n-1], ofs+1)
224 | 				f.write([]byte{v - 0xfe}, ofs+atoms<<4-1)
225 | 				return
226 | 			}
227 | 			endmark = false
228 | 		}
229 | 		// non esacpe
230 | 		pre := []byte{byte(n)}
231 | 		f.write(pre, ofs)
232 | 		f.write(b, ofs+1)
233 | 		if endmark {
234 | 			f.write(zero, ofs+atoms<<4-1) // last block byte <- used block
235 | 		}
236 | 	case n > 237 && n <= 61680:
237 | 		if (n+3)&0xf == 0 { // content end == atom end
238 | 			if v := b[n-1]; v >= 0xfe { // escape
239 | 				x := (16*0xf0f1 + n - 13) >> 4
240 | 				pre := []byte{0xFC, byte(x >> 8), byte(x)}
241 | 				f.write(pre, ofs)
242 | 				f.write(b[:n-1], ofs+3)
243 | 				f.write([]byte{v - 0xfe}, ofs+atoms<<4-1)
244 | 				return
245 | 			}
246 | 			endmark = false
247 | 		}
248 | 		// non esacpe
249 | 		pre := []byte{0xfc, byte(n >> 8), byte(n)}
250 | 		f.write(pre, ofs)
251 | 		f.write(b, ofs+3)
252 | 		if endmark {
253 | 			f.write(zero, ofs+atoms<<4-1) // last block byte <- used block
254 | 		}
255 | 	}
256 | }
257 | 
258 | func rq2Atoms(rqbytes int) (rqatoms int64) {
259 | 	if rqbytes > 237 {
260 | 		rqbytes += 2
261 | 	}
262 | 	return int64(rqbytes>>4 + 1)
263 | }
264 | 
265 | func (f *File) extend(b []byte) (handle int64) {
266 | 	handle = f.atoms
267 | 	f.writeUsed(b, handle)
268 | 	f.atoms += rq2Atoms(len(b))
269 | 	return
270 | }
271 | 
272 | // Alloc stores b in a newly allocated space and returns its handle and an error if any.
273 | func (f *File) Alloc(b []byte) (handle Handle, err error) {
274 | 	err = storage.Mutate(f.Accessor(), func() (err error) {
275 | 		rqAtoms := rq2Atoms(len(b))
276 | 		if rqAtoms > 3856 {
277 | 			return &EBadRequest{f.f.Name(), len(b)}
278 | 		}
279 | 
280 | 		for foundsize, foundp := range f.freetab[rqAtoms:] {
281 | 			if foundp != 0 {
282 | 				// this works only for the current unique sizes list (except the last item!)
283 | 				size := int64(foundsize) + rqAtoms
284 | 				handle = Handle(foundp)
285 | 				if size == 3856 {
286 | 					buf := make([]byte, 7)
287 | 					f.read(buf, int64(handle)<<4+15)
288 | 					(*Handle)(&size).Get(buf)
289 | 				}
290 | 				f.delFree(int64(handle), size)
291 | 				if rqAtoms < size {
292 | 					f.addFree(int64(handle)+rqAtoms, size-rqAtoms)
293 | 				}
294 | 				f.writeUsed(b, int64(handle))
295 | 				return
296 | 			}
297 | 		}
298 | 
299 | 		handle = Handle(f.extend(b))
300 | 		return
301 | 	})
302 | 	return
303 | }
304 | 
305 | // checkLeft returns the atom size of a free bleck left adjacent to block @atom.
306 | // If that block is not free the returned size is 0.
307 | func (f *File) checkLeft(atom int64) (size int64) {
308 | 	if atom <= f.canfree {
309 | 		return
310 | 	}
311 | 
312 | 	b := make([]byte, 7)
313 | 	fp := atom << 4
314 | 	f.read(b[:1], fp-1)
315 | 	switch last := b[0]; {
316 | 	case last <= 0xfd:
317 | 		// used block
318 | 	case last == 0xfe:
319 | 		f.read(b, fp-8)
320 | 		(*Handle)(&size).Get(b)
321 | 	case last == 0xff:
322 | 		size = 1
323 | 	}
324 | 	return
325 | }
326 | 
327 | // getInfo returns the block @atom type and size.
328 | func (f *File) getInfo(atom int64) (pref byte, size int64) {
329 | 	b := make([]byte, 7)
330 | 	fp := atom << 4
331 | 	f.read(b[:1], fp)
332 | 	switch pref = b[0]; {
333 | 	case pref == 0: // Empty used
334 | 		size = 1
335 | 	case pref >= 1 && pref <= 237: // Short
336 | 		size = rq2Atoms(int(pref))
337 | 	case pref >= 0xee && pref <= 0xfb: // Short esc
338 | 		size = rq2Atoms(15 + 16*int(pref-0xee))
339 | 	case pref == 0xfc: // Long
340 | 		f.read(b[:2], fp+1)
341 | 		n := int(b[0])<<8 + int(b[1])
342 | 		switch {
343 | 		default:
344 | 			panic(&ECorrupted{f.f.Name(), fp + 1})
345 | 		case n >= 238 && n <= 61680: // Long non esc
346 | 			size = rq2Atoms(n)
347 | 		case n >= 61681: // Long esc
348 | 			size = rq2Atoms(13 + 16*(n-0xf0f1))
349 | 		}
350 | 	case pref == 0xfd: // reloc
351 | 		size = 1
352 | 	case pref == 0xfe:
353 | 		f.read(b, fp+15)
354 | 		(*Handle)(&size).Get(b)
355 | 	case pref == 0xff:
356 | 		size = 1
357 | 	}
358 | 	return
359 | }
360 | 
361 | // getSize returns the atom size of the block @atom and wheter it is free.
362 | func (f *File) getSize(atom int64) (size int64, isFree bool) {
363 | 	var typ byte
364 | 	typ, size = f.getInfo(atom)
365 | 	isFree = typ >= 0xfe
366 | 	return
367 | }
368 | 
369 | // checkRight returns the atom size of a free bleck right adjacent to block @atom,atoms.
370 | // If that block is not free the returned size is 0.
371 | func (f *File) checkRight(atom, atoms int64) (size int64) {
372 | 	if atom+atoms >= f.atoms {
373 | 		return
374 | 	}
375 | 
376 | 	if sz, free := f.getSize(atom + atoms); free {
377 | 		size = sz
378 | 	}
379 | 	return
380 | }
381 | 
382 | // delFree removes the atoms@atom free block from the free block list
383 | func (f *File) delFree(atom, atoms int64) {
384 | 	b := make([]byte, 15)
385 | 	size := int(atoms)
386 | 	if n := len(f.freetab); atoms >= int64(n) {
387 | 		size = n - 1
388 | 	}
389 | 	fp := atom << 4
390 | 	f.read(b[1:], fp+1)
391 | 	var prev, next Handle
392 | 	prev.Get(b[1:])
393 | 	next.Get(b[8:])
394 | 
395 | 	switch {
396 | 	case prev == 0 && next != 0:
397 | 		next.Put(b)
398 | 		f.write(b[:7], int64(32+3+7+(size-1)*14))
399 | 		f.write(zero7, int64(next)<<4+1)
400 | 		f.freetab[size] = int64(next)
401 | 	case prev != 0 && next == 0:
402 | 		f.write(zero7, int64(prev)<<4+8)
403 | 	case prev != 0 && next != 0:
404 | 		prev.Put(b)
405 | 		f.write(b[:7], int64(next)<<4+1)
406 | 		next.Put(b)
407 | 		f.write(b[:7], int64(prev)<<4+8)
408 | 	default: // prev == 0 && next == 0:
409 | 		f.write(zero7, int64(32+3+7+(size-1)*14))
410 | 		f.freetab[size] = 0
411 | 	}
412 | }
413 | 
414 | // addFree adds atoms@atom to the free block lists and marks it free.
415 | func (f *File) addFree(atom, atoms int64) {
416 | 	b := make([]byte, 7)
417 | 	size := int(atoms)
418 | 	if n := len(f.freetab); atoms >= int64(n) {
419 | 		size = n - 1
420 | 	}
421 | 	head := f.freetab[size]
422 | 	if head == 0 { // empty list
423 | 		f.makeFree(0, atom, atoms, 0)
424 | 		Handle(atom).Put(b)
425 | 		f.write(b, int64(32+3+7+(size-1)*14))
426 | 		f.freetab[size] = atom
427 | 		return
428 | 	}
429 | 
430 | 	Handle(atom).Put(b)
431 | 	f.write(b, head<<4+1)            // head.prev = atom
432 | 	f.makeFree(0, atom, atoms, head) // atom.next = head
433 | 	f.write(b, int64(32+3+7+(size-1)*14))
434 | 	f.freetab[size] = atom
435 | }
436 | 
437 | // makeFree sets up the content of a free block atoms@atom, fills the prev and next links.
438 | func (f *File) makeFree(prev, atom, atoms, next int64) {
439 | 	b := make([]byte, 23)
440 | 	fp := atom << 4
441 | 	if atoms == 1 {
442 | 		b[0] = 0xff
443 | 		Handle(prev).Put(b[1:])
444 | 		Handle(next).Put(b[8:])
445 | 		b[15] = 0xff
446 | 		f.write(b[:16], fp)
447 | 		return
448 | 	}
449 | 
450 | 	b[0] = 0xfe
451 | 	Handle(prev).Put(b[1:])
452 | 	Handle(next).Put(b[8:])
453 | 	Handle(atoms).Put(b[15:])
454 | 	f.write(b[:22], fp)
455 | 	b[22] = 0xfe
456 | 	f.write(b[15:], fp+atoms<<4-8)
457 | }
458 | 
459 | // Read reads and return the data associated with handle and an error if any.
460 | // Passing an invalid handle to Read may return invalid data without error.
461 | // It's like getting garbage via passing an invalid pointer to C.memcopy().
462 | func (f *File) Read(handle Handle) (b []byte, err error) {
463 | 	defer func() {
464 | 		if e := recover(); e != nil {
465 | 			b = nil
466 | 			err = e.(error)
467 | 		}
468 | 	}()
469 | 
470 | 	switch handle {
471 | 	case 0:
472 | 		panic(ENullHandle(f.f.Name()))
473 | 	case 2:
474 | 		panic(&EHandle{f.f.Name(), handle})
475 | 	default:
476 | 		b, _ = f.readUsed(int64(handle))
477 | 	}
478 | 	return
479 | }
480 | 
481 | // Free frees space associated with handle and returns an error if any. Passing an invalid
482 | // handle to Free or reusing handle afterwards will probably corrupt the database or provide
483 | // invalid data on Read. It's like corrupting memory via passing an invalid pointer to C.free()
484 | // or reusing that pointer.
485 | func (f *File) Free(handle Handle) (err error) {
486 | 	return storage.Mutate(f.Accessor(), func() (err error) {
487 | 		atom := int64(handle)
488 | 		atoms, isFree := f.getSize(atom)
489 | 		if isFree || atom < f.canfree {
490 | 			return &EHandle{f.f.Name(), handle}
491 | 		}
492 | 
493 | 		leftFree, rightFree := f.checkLeft(atom), f.checkRight(atom, atoms)
494 | 		switch {
495 | 		case leftFree != 0 && rightFree != 0:
496 | 			f.delFree(atom-leftFree, leftFree)
497 | 			f.delFree(atom+atoms, rightFree)
498 | 			f.addFree(atom-leftFree, leftFree+atoms+rightFree)
499 | 		case leftFree != 0 && rightFree == 0:
500 | 			f.delFree(atom-leftFree, leftFree)
501 | 			if atom+atoms == f.atoms { // the left free neighbour and this block together are an empy tail
502 | 				f.atoms = atom - leftFree
503 | 				f.f.Truncate(f.atoms << 4)
504 | 				return
505 | 			}
506 | 
507 | 			f.addFree(atom-leftFree, leftFree+atoms)
508 | 		case leftFree == 0 && rightFree != 0:
509 | 			f.delFree(atom+atoms, rightFree)
510 | 			f.addFree(atom, atoms+rightFree)
511 | 		default: // leftFree == 0 && rightFree == 0
512 | 			if atom+atoms < f.atoms { // isolated inner block
513 | 				f.addFree(atom, atoms)
514 | 				return
515 | 			}
516 | 
517 | 			f.f.Truncate(atom << 4) // isolated tail block, shrink file
518 | 			f.atoms = atom
519 | 		}
520 | 		return
521 | 	})
522 | }
523 | 
524 | // Realloc reallocates space associted with handle to acomodate b, returns the newhandle
525 | // newly associated with b and an error if any. If keepHandle == true then Realloc guarantees
526 | // newhandle == handle even if the new data are larger then the previous content associated
527 | // with handle. If !keepHandle && newhandle != handle then reusing handle will probably corrupt
528 | // the database.
529 | // The above effects are like corrupting memory/data via passing an invalid pointer to C.realloc().
530 | func (f *File) Realloc(handle Handle, b []byte, keepHandle bool) (newhandle Handle, err error) {
531 | 	err = storage.Mutate(f.Accessor(), func() (err error) {
532 | 		switch handle {
533 | 		case 0, 2:
534 | 			return &EHandle{f.f.Name(), handle}
535 | 		case 1:
536 | 			keepHandle = true
537 | 		}
538 | 		newhandle = handle
539 | 		atom, newatoms := int64(handle), rq2Atoms(len(b))
540 | 		if newatoms > 3856 {
541 | 			return &EBadRequest{f.f.Name(), len(b)}
542 | 		}
543 | 
544 | 		typ, oldatoms := f.getInfo(atom)
545 | 		switch {
546 | 		default:
547 | 			return &ECorrupted{f.f.Name(), atom << 4}
548 | 		case typ <= 0xfc: // non relocated used block
549 | 			switch {
550 | 			case newatoms == oldatoms: // in place replace
551 | 				f.writeUsed(b, atom)
552 | 			case newatoms < oldatoms: // in place shrink
553 | 				rightFree := f.checkRight(atom, oldatoms)
554 | 				if rightFree > 0 { // right join
555 | 					f.delFree(atom+oldatoms, rightFree)
556 | 				}
557 | 				f.addFree(atom+newatoms, oldatoms+rightFree-newatoms)
558 | 				f.writeUsed(b, atom)
559 | 			case newatoms > oldatoms:
560 | 				if rightFree := f.checkRight(atom, oldatoms); rightFree > 0 && newatoms <= oldatoms+rightFree {
561 | 					f.delFree(atom+oldatoms, rightFree)
562 | 					if newatoms < oldatoms+rightFree {
563 | 						f.addFree(atom+newatoms, oldatoms+rightFree-newatoms)
564 | 					}
565 | 					f.writeUsed(b, atom)
566 | 					return
567 | 				}
568 | 
569 | 				if !keepHandle {
570 | 					f.Free(Handle(atom))
571 | 					newhandle, err = f.Alloc(b)
572 | 					return
573 | 				}
574 | 
575 | 				// reloc
576 | 				newatom, e := f.Alloc(b)
577 | 				if e != nil {
578 | 					return e
579 | 				}
580 | 
581 | 				buf := make([]byte, 16)
582 | 				buf[0] = 0xfd
583 | 				Handle(newatom).Put(buf[1:])
584 | 				f.Realloc(Handle(atom), buf[1:], true)
585 | 				f.write(buf[:1], atom<<4)
586 | 			}
587 | 		case typ == 0xfd: // reloc
588 | 			var target Handle
589 | 			buf := make([]byte, 7)
590 | 			f.read(buf, atom<<4+1)
591 | 			target.Get(buf)
592 | 			switch {
593 | 			case newatoms == 1:
594 | 				f.writeUsed(b, atom)
595 | 				f.Free(target)
596 | 			default:
597 | 				if rightFree := f.checkRight(atom, 1); rightFree > 0 && newatoms <= 1+rightFree {
598 | 					f.delFree(atom+1, rightFree)
599 | 					if newatoms < 1+rightFree {
600 | 						f.addFree(atom+newatoms, 1+rightFree-newatoms)
601 | 					}
602 | 					f.writeUsed(b, atom)
603 | 					f.Free(target)
604 | 					return
605 | 				}
606 | 
607 | 				newtarget, e := f.Realloc(Handle(target), b, false)
608 | 				if e != nil {
609 | 					return e
610 | 				}
611 | 
612 | 				if newtarget != target {
613 | 					Handle(newtarget).Put(buf)
614 | 					f.write(buf, atom<<4+1)
615 | 				}
616 | 			}
617 | 		}
618 | 		return
619 | 	})
620 | 	return
621 | }
622 | 
623 | // Lock locks f for writing. If the lock is already locked for reading or writing,
624 | // Lock blocks until the lock is available. To ensure that the lock eventually becomes available,
625 | // a blocked Lock call excludes new readers from acquiring the lock.
626 | func (f *File) Lock() {
627 | 	f.rwm.Lock()
628 | }
629 | 
630 | // RLock locks f for reading. If the lock is already locked for writing or there is a writer
631 | // already waiting to release the lock, RLock blocks until the writer has released the lock.
632 | func (f *File) RLock() {
633 | 	f.rwm.RLock()
634 | }
635 | 
636 | // Unlock unlocks f for writing. It is a run-time error if f is not locked for writing on entry to Unlock.
637 | //
638 | // As with Mutexes, a locked RWMutex is not associated with a particular goroutine.
639 | // One goroutine may RLock (Lock) f and then arrange for another goroutine to RUnlock (Unlock) it.
640 | func (f *File) Unlock() {
641 | 	f.rwm.Unlock()
642 | }
643 | 
644 | // RUnlock undoes a single RLock call; it does not affect other simultaneous readers.
645 | // It is a run-time error if f is not locked for reading on entry to RUnlock.
646 | func (f *File) RUnlock() {
647 | 	f.rwm.RUnlock()
648 | }
649 | 
650 | // LockedAlloc wraps Alloc in a Lock/Unlock pair.
651 | func (f *File) LockedAlloc(b []byte) (handle Handle, err error) {
652 | 	f.Lock()
653 | 	defer f.Unlock()
654 | 	return f.Alloc(b)
655 | }
656 | 
657 | // LockedFree wraps Free in a Lock/Unlock pair.
658 | func (f *File) LockedFree(handle Handle) (err error) {
659 | 	f.Lock()
660 | 	defer f.Unlock()
661 | 	return f.Free(handle)
662 | }
663 | 
664 | // LockedRead wraps Read in a RLock/RUnlock pair.
665 | func (f *File) LockedRead(handle Handle) (b []byte, err error) {
666 | 	f.RLock()
667 | 	defer f.RUnlock()
668 | 	return f.Read(handle)
669 | }
670 | 
671 | // LockedRealloc wraps Realloc in a Lock/Unlock pair.
672 | func (f *File) LockedRealloc(handle Handle, b []byte, keepHandle bool) (newhandle Handle, err error) {
673 | 	f.Lock()
674 | 	defer f.Unlock()
675 | 	return f.Realloc(handle, b, keepHandle)
676 | }
677 | 


--------------------------------------------------------------------------------
/falloc/test_deps.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package falloc
 8 | 
 9 | // Pull test dependencies too.
10 | // Enables easy 'go test X' after 'go get X'
11 | import (
12 | 	_ "github.com/cznic/fileutil"
13 | 	_ "github.com/cznic/fileutil/storage"
14 | 	_ "github.com/cznic/mathutil"
15 | )
16 | 


--------------------------------------------------------------------------------
/fileutil.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // Package fileutil collects some file utility functions.
  6 | package fileutil
  7 | 
  8 | import (
  9 | 	"fmt"
 10 | 	"io"
 11 | 	"os"
 12 | 	"path/filepath"
 13 | 	"runtime"
 14 | 	"strconv"
 15 | 	"sync"
 16 | 	"time"
 17 | )
 18 | 
 19 | // GoMFile is a concurrent access safe version of MFile.
 20 | type GoMFile struct {
 21 | 	mfile *MFile
 22 | 	mutex sync.Mutex
 23 | }
 24 | 
 25 | // NewGoMFile return a newly created GoMFile.
 26 | func NewGoMFile(fname string, flag int, perm os.FileMode, delta_ns int64) (m *GoMFile, err error) {
 27 | 	m = &GoMFile{}
 28 | 	if m.mfile, err = NewMFile(fname, flag, perm, delta_ns); err != nil {
 29 | 		m = nil
 30 | 	}
 31 | 	return
 32 | }
 33 | 
 34 | func (m *GoMFile) File() (file *os.File, err error) {
 35 | 	m.mutex.Lock()
 36 | 	defer m.mutex.Unlock()
 37 | 	return m.mfile.File()
 38 | }
 39 | 
 40 | func (m *GoMFile) SetChanged() {
 41 | 	m.mutex.Lock()
 42 | 	defer m.mutex.Unlock()
 43 | 	m.mfile.SetChanged()
 44 | }
 45 | 
 46 | func (m *GoMFile) SetHandler(h MFileHandler) {
 47 | 	m.mutex.Lock()
 48 | 	defer m.mutex.Unlock()
 49 | 	m.mfile.SetHandler(h)
 50 | }
 51 | 
 52 | // MFileHandler resolves modifications of File.
 53 | // Possible File context is expected to be a part of the handler's closure.
 54 | type MFileHandler func(*os.File) error
 55 | 
 56 | // MFile represents an os.File with a guard/handler on change/modification.
 57 | // Example use case is an app with a configuration file which can be modified at any time
 58 | // and have to be reloaded in such event prior to performing something configurable by that
 59 | // file. The checks are made only on access to the MFile file by
 60 | // File() and a time threshold/hysteresis value can be chosen on creating a new MFile.
 61 | type MFile struct {
 62 | 	file    *os.File
 63 | 	handler MFileHandler
 64 | 	t0      int64
 65 | 	delta   int64
 66 | 	ctime   int64
 67 | }
 68 | 
 69 | // NewMFile returns a newly created MFile or Error if any.
 70 | // The fname, flag and perm parameters have the same meaning as in os.Open.
 71 | // For meaning of the delta_ns parameter please see the (m *MFile) File() docs.
 72 | func NewMFile(fname string, flag int, perm os.FileMode, delta_ns int64) (m *MFile, err error) {
 73 | 	m = &MFile{}
 74 | 	m.t0 = time.Now().UnixNano()
 75 | 	if m.file, err = os.OpenFile(fname, flag, perm); err != nil {
 76 | 		return
 77 | 	}
 78 | 
 79 | 	var fi os.FileInfo
 80 | 	if fi, err = m.file.Stat(); err != nil {
 81 | 		return
 82 | 	}
 83 | 
 84 | 	m.ctime = fi.ModTime().UnixNano()
 85 | 	m.delta = delta_ns
 86 | 	runtime.SetFinalizer(m, func(m *MFile) {
 87 | 		m.file.Close()
 88 | 	})
 89 | 	return
 90 | }
 91 | 
 92 | // SetChanged forces next File() to unconditionally handle modification of the wrapped os.File.
 93 | func (m *MFile) SetChanged() {
 94 | 	m.ctime = -1
 95 | }
 96 | 
 97 | // SetHandler sets a function to be invoked when modification of MFile is to be processed.
 98 | func (m *MFile) SetHandler(h MFileHandler) {
 99 | 	m.handler = h
100 | }
101 | 
102 | // File returns an os.File from MFile. If time elapsed between the last invocation of this function
103 | // and now is at least delta_ns ns (a parameter of NewMFile) then the file is checked for
104 | // change/modification. For delta_ns == 0 the modification is checked w/o getting os.Time().
105 | // If a change is detected a handler is invoked on the MFile file.
106 | // Any of these steps can produce an Error. If that happens the function returns nil, Error.
107 | func (m *MFile) File() (file *os.File, err error) {
108 | 	var now int64
109 | 
110 | 	mustCheck := m.delta == 0
111 | 	if !mustCheck {
112 | 		now = time.Now().UnixNano()
113 | 		mustCheck = now-m.t0 > m.delta
114 | 	}
115 | 
116 | 	if mustCheck { // check interval reached
117 | 		var fi os.FileInfo
118 | 		if fi, err = m.file.Stat(); err != nil {
119 | 			return
120 | 		}
121 | 
122 | 		if fi.ModTime().UnixNano() != m.ctime { // modification detected
123 | 			if m.handler == nil {
124 | 				return nil, fmt.Errorf("no handler set for modified file %q", m.file.Name())
125 | 			}
126 | 			if err = m.handler(m.file); err != nil {
127 | 				return
128 | 			}
129 | 
130 | 			m.ctime = fi.ModTime().UnixNano()
131 | 		}
132 | 		m.t0 = now
133 | 	}
134 | 
135 | 	return m.file, nil
136 | }
137 | 
138 | // Read reads buf from r. It will either fill the full buf or fail.
139 | // It wraps the functionality of an io.Reader which may return less bytes than requested,
140 | // but may block if not all data are ready for the io.Reader.
141 | func Read(r io.Reader, buf []byte) (err error) {
142 | 	have := 0
143 | 	remain := len(buf)
144 | 	got := 0
145 | 	for remain > 0 {
146 | 		if got, err = r.Read(buf[have:]); err != nil {
147 | 			return
148 | 		}
149 | 
150 | 		remain -= got
151 | 		have += got
152 | 	}
153 | 	return
154 | }
155 | 
156 | // "os" and/or "syscall" extensions
157 | 
158 | // FadviseAdvice is used by Fadvise.
159 | type FadviseAdvice int
160 | 
161 | // FAdviseAdvice values.
162 | const (
163 | 	// $ grep FADV /usr/include/bits/fcntl.h
164 | 	POSIX_FADV_NORMAL     FadviseAdvice = iota // No further special treatment.
165 | 	POSIX_FADV_RANDOM                          // Expect random page references.
166 | 	POSIX_FADV_SEQUENTIAL                      // Expect sequential page references.
167 | 	POSIX_FADV_WILLNEED                        // Will need these pages.
168 | 	POSIX_FADV_DONTNEED                        // Don't need these pages.
169 | 	POSIX_FADV_NOREUSE                         // Data will be accessed once.
170 | )
171 | 
172 | // TempFile creates a new temporary file in the directory dir with a name
173 | // ending with suffix, basename starting with prefix, opens the file for
174 | // reading and writing, and returns the resulting *os.File.  If dir is the
175 | // empty string, TempFile uses the default directory for temporary files (see
176 | // os.TempDir).  Multiple programs calling TempFile simultaneously will not
177 | // choose the same file.  The caller can use f.Name() to find the pathname of
178 | // the file.  It is the caller's responsibility to remove the file when no
179 | // longer needed.
180 | //
181 | // NOTE: This function differs from ioutil.TempFile.
182 | func TempFile(dir, prefix, suffix string) (f *os.File, err error) {
183 | 	if dir == "" {
184 | 		dir = os.TempDir()
185 | 	}
186 | 
187 | 	nconflict := 0
188 | 	for i := 0; i < 10000; i++ {
189 | 		name := filepath.Join(dir, prefix+nextInfix()+suffix)
190 | 		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
191 | 		if os.IsExist(err) {
192 | 			if nconflict++; nconflict > 10 {
193 | 				randmu.Lock()
194 | 				rand = reseed()
195 | 				randmu.Unlock()
196 | 			}
197 | 			continue
198 | 		}
199 | 		break
200 | 	}
201 | 	return
202 | }
203 | 
204 | // Random number state.
205 | // We generate random temporary file names so that there's a good
206 | // chance the file doesn't exist yet - keeps the number of tries in
207 | // TempFile to a minimum.
208 | var rand uint32
209 | var randmu sync.Mutex
210 | 
211 | func reseed() uint32 {
212 | 	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
213 | }
214 | 
215 | func nextInfix() string {
216 | 	randmu.Lock()
217 | 	r := rand
218 | 	if r == 0 {
219 | 		r = reseed()
220 | 	}
221 | 	r = r*1664525 + 1013904223 // constants from Numerical Recipes
222 | 	rand = r
223 | 	randmu.Unlock()
224 | 	return strconv.Itoa(int(1e9 + r%1e9))[1:]
225 | }
226 | 


--------------------------------------------------------------------------------
/fileutil_arm.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package fileutil
 6 | 
 7 | import (
 8 | 	"io"
 9 | 	"os"
10 | )
11 | 
12 | const hasPunchHole = false
13 | 
14 | // PunchHole deallocates space inside a file in the byte range starting at
15 | // offset and continuing for len bytes. Not supported on ARM.
16 | func PunchHole(f *os.File, off, len int64) error {
17 | 	return nil
18 | }
19 | 
20 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
21 | // posix_fadvise'. Not supported on ARM.
22 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
23 | 	return nil
24 | }
25 | 
26 | // IsEOF reports whether err is an EOF condition.
27 | func IsEOF(err error) bool { return err == io.EOF }
28 | 


--------------------------------------------------------------------------------
/fileutil_darwin.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build !arm
 6 | 
 7 | package fileutil
 8 | 
 9 | import (
10 | 	"io"
11 | 	"os"
12 | )
13 | 
14 | const hasPunchHole = false
15 | 
16 | // PunchHole deallocates space inside a file in the byte range starting at
17 | // offset and continuing for len bytes. Not supported on OSX.
18 | func PunchHole(f *os.File, off, len int64) error {
19 | 	return nil
20 | }
21 | 
22 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
23 | // posix_fadvise'. Not supported on OSX.
24 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
25 | 	return nil
26 | }
27 | 
28 | // IsEOF reports whether err is an EOF condition.
29 | func IsEOF(err error) bool { return err == io.EOF }
30 | 


--------------------------------------------------------------------------------
/fileutil_dragonfly.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build !arm
 6 | 
 7 | package fileutil
 8 | 
 9 | import (
10 | 	"io"
11 | 	"os"
12 | )
13 | 
14 | const hasPunchHole = false
15 | 
16 | // PunchHole deallocates space inside a file in the byte range starting at
17 | // offset and continuing for len bytes. Unimplemented on DragonFlyBSD.
18 | func PunchHole(f *os.File, off, len int64) error {
19 | 	return nil
20 | }
21 | 
22 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
23 | // posix_fadvise'. Unimplemented on DragonFlyBSD.
24 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
25 | 	return nil
26 | }
27 | 
28 | // IsEOF reports whether err is an EOF condition.
29 | func IsEOF(err error) bool { return err == io.EOF }
30 | 


--------------------------------------------------------------------------------
/fileutil_freebsd.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build !arm
 6 | 
 7 | package fileutil
 8 | 
 9 | import (
10 | 	"io"
11 | 	"os"
12 | )
13 | 
14 | const hasPunchHole = false
15 | 
16 | // PunchHole deallocates space inside a file in the byte range starting at
17 | // offset and continuing for len bytes. Unimplemented on FreeBSD.
18 | func PunchHole(f *os.File, off, len int64) error {
19 | 	return nil
20 | }
21 | 
22 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
23 | // posix_fadvise'. Unimplemented on FreeBSD.
24 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
25 | 	return nil
26 | }
27 | 
28 | // IsEOF reports whether err is an EOF condition.
29 | func IsEOF(err error) bool { return err == io.EOF }
30 | 


--------------------------------------------------------------------------------
/fileutil_linux.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build !arm
 6 | 
 7 | package fileutil
 8 | 
 9 | import (
10 | 	"bytes"
11 | 	"io"
12 | 	"io/ioutil"
13 | 	"os"
14 | 	"strconv"
15 | 	"syscall"
16 | )
17 | 
18 | const hasPunchHole = true
19 | 
20 | func n(s []byte) byte {
21 | 	for i, c := range s {
22 | 		if c < '0' || c > '9' {
23 | 			s = s[:i]
24 | 			break
25 | 		}
26 | 	}
27 | 	v, _ := strconv.Atoi(string(s))
28 | 	return byte(v)
29 | }
30 | 
31 | func init() {
32 | 	b, err := ioutil.ReadFile("/proc/sys/kernel/osrelease")
33 | 	if err != nil {
34 | 		panic(err)
35 | 	}
36 | 
37 | 	tokens := bytes.Split(b, []byte("."))
38 | 	if len(tokens) > 3 {
39 | 		tokens = tokens[:3]
40 | 	}
41 | 	switch len(tokens) {
42 | 	case 3:
43 | 		// Supported since kernel 2.6.38
44 | 		if bytes.Compare([]byte{n(tokens[0]), n(tokens[1]), n(tokens[2])}, []byte{2, 6, 38}) < 0 {
45 | 			puncher = func(*os.File, int64, int64) error { return nil }
46 | 		}
47 | 	case 2:
48 | 		if bytes.Compare([]byte{n(tokens[0]), n(tokens[1])}, []byte{2, 7}) < 0 {
49 | 			puncher = func(*os.File, int64, int64) error { return nil }
50 | 		}
51 | 	default:
52 | 		puncher = func(*os.File, int64, int64) error { return nil }
53 | 	}
54 | }
55 | 
56 | var puncher = func(f *os.File, off, len int64) error {
57 | 	const (
58 | 		/*
59 | 			/usr/include/linux$ grep FL_ falloc.h
60 | 		*/
61 | 		_FALLOC_FL_KEEP_SIZE  = 0x01 // default is extend size
62 | 		_FALLOC_FL_PUNCH_HOLE = 0x02 // de-allocates range
63 | 	)
64 | 
65 | 	_, _, errno := syscall.Syscall6(
66 | 		syscall.SYS_FALLOCATE,
67 | 		uintptr(f.Fd()),
68 | 		uintptr(_FALLOC_FL_KEEP_SIZE|_FALLOC_FL_PUNCH_HOLE),
69 | 		uintptr(off),
70 | 		uintptr(len),
71 | 		0, 0)
72 | 	if errno != 0 {
73 | 		return os.NewSyscallError("SYS_FALLOCATE", errno)
74 | 	}
75 | 	return nil
76 | }
77 | 
78 | // PunchHole deallocates space inside a file in the byte range starting at
79 | // offset and continuing for len bytes. No-op for kernels < 2.6.38 (or < 2.7).
80 | func PunchHole(f *os.File, off, len int64) error {
81 | 	return puncher(f, off, len)
82 | }
83 | 
84 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
85 | // posix_fadvise'.
86 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
87 | 	_, _, errno := syscall.Syscall6(
88 | 		syscall.SYS_FADVISE64,
89 | 		uintptr(f.Fd()),
90 | 		uintptr(off),
91 | 		uintptr(len),
92 | 		uintptr(advice),
93 | 		0, 0)
94 | 	return os.NewSyscallError("SYS_FADVISE64", errno)
95 | }
96 | 
97 | // IsEOF reports whether err is an EOF condition.
98 | func IsEOF(err error) bool { return err == io.EOF }
99 | 


--------------------------------------------------------------------------------
/fileutil_netbsd.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build !arm
 6 | 
 7 | package fileutil
 8 | 
 9 | import (
10 | 	"io"
11 | 	"os"
12 | )
13 | 
14 | const hasPunchHole = false
15 | 
16 | // PunchHole deallocates space inside a file in the byte range starting at
17 | // offset and continuing for len bytes. Similar to FreeBSD, this is
18 | // unimplemented.
19 | func PunchHole(f *os.File, off, len int64) error {
20 | 	return nil
21 | }
22 | 
23 | // Unimplemented on NetBSD.
24 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
25 | 	return nil
26 | }
27 | 
28 | // IsEOF reports whether err is an EOF condition.
29 | func IsEOF(err error) bool { return err == io.EOF }
30 | 


--------------------------------------------------------------------------------
/fileutil_openbsd.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package fileutil
 6 | 
 7 | import (
 8 | 	"io"
 9 | 	"os"
10 | )
11 | 
12 | const hasPunchHole = false
13 | 
14 | // PunchHole deallocates space inside a file in the byte range starting at
15 | // offset and continuing for len bytes. Similar to FreeBSD, this is
16 | // unimplemented.
17 | func PunchHole(f *os.File, off, len int64) error {
18 | 	return nil
19 | }
20 | 
21 | // Unimplemented on OpenBSD.
22 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
23 | 	return nil
24 | }
25 | 
26 | // IsEOF reports whether err is an EOF condition.
27 | func IsEOF(err error) bool { return err == io.EOF }
28 | 


--------------------------------------------------------------------------------
/fileutil_plan9.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package fileutil
 6 | 
 7 | import (
 8 | 	"io"
 9 | 	"os"
10 | )
11 | 
12 | const hasPunchHole = false
13 | 
14 | // PunchHole deallocates space inside a file in the byte range starting at
15 | // offset and continuing for len bytes. Unimplemented on Plan 9.
16 | func PunchHole(f *os.File, off, len int64) error {
17 | 	return nil
18 | }
19 | 
20 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
21 | // posix_fadvise'. Unimplemented on Plan 9.
22 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
23 | 	return nil
24 | }
25 | 
26 | // IsEOF reports whether err is an EOF condition.
27 | func IsEOF(err error) bool { return err == io.EOF }
28 | 


--------------------------------------------------------------------------------
/fileutil_solaris.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2013 jnml. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build go1.3
 6 | 
 7 | package fileutil
 8 | 
 9 | import (
10 | 	"io"
11 | 	"os"
12 | )
13 | 
14 | const hasPunchHole = false
15 | 
16 | // PunchHole deallocates space inside a file in the byte range starting at
17 | // offset and continuing for len bytes. Not supported on Solaris.
18 | func PunchHole(f *os.File, off, len int64) error {
19 | 	return nil
20 | }
21 | 
22 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
23 | // posix_fadvise'. Not supported on Solaris.
24 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
25 | 	return nil
26 | }
27 | 
28 | // IsEOF reports whether err is an EOF condition.
29 | func IsEOF(err error) bool { return err == io.EOF }
30 | 


--------------------------------------------------------------------------------
/fileutil_windows.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | package fileutil
  6 | 
  7 | import (
  8 | 	"io"
  9 | 	"os"
 10 | 	"sync"
 11 | 	"syscall"
 12 | 	"unsafe"
 13 | )
 14 | 
 15 | const hasPunchHole = true
 16 | 
 17 | // PunchHole deallocates space inside a file in the byte range starting at
 18 | // offset and continuing for len bytes. Not supported on Windows.
 19 | func PunchHole(f *os.File, off, len int64) error {
 20 | 	return puncher(f, off, len)
 21 | }
 22 | 
 23 | // Fadvise predeclares an access pattern for file data.  See also 'man 2
 24 | // posix_fadvise'. Not supported on Windows.
 25 | func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
 26 | 	return nil
 27 | }
 28 | 
 29 | // IsEOF reports whether err is an EOF condition.
 30 | func IsEOF(err error) bool {
 31 | 	if err == io.EOF {
 32 | 		return true
 33 | 	}
 34 | 
 35 | 	// http://social.technet.microsoft.com/Forums/windowsserver/en-US/1a16311b-c625-46cf-830b-6a26af488435/how-to-solve-error-38-0x26-errorhandleeof-using-fsctlgetretrievalpointers
 36 | 	x, ok := err.(*os.PathError)
 37 | 	return ok && x.Op == "read" && x.Err.(syscall.Errno) == 0x26
 38 | }
 39 | 
 40 | var (
 41 | 	modkernel32 = syscall.NewLazyDLL("kernel32.dll")
 42 | 
 43 | 	procDeviceIOControl = modkernel32.NewProc("DeviceIoControl")
 44 | 
 45 | 	sparseFilesMu sync.Mutex
 46 | 	sparseFiles   map[uintptr]struct{}
 47 | )
 48 | 
 49 | func init() {
 50 | 	// sparseFiles is an fd set for already "sparsed" files - according to
 51 | 	// msdn.microsoft.com/en-us/library/windows/desktop/aa364225(v=vs.85).aspx
 52 | 	// the file handles are unique per process.
 53 | 	sparseFiles = make(map[uintptr]struct{})
 54 | }
 55 | 
 56 | // puncHoleWindows punches a hole into the given file starting at offset,
 57 | // measuring "size" bytes
 58 | // (http://msdn.microsoft.com/en-us/library/windows/desktop/aa364597%28v=vs.85%29.aspx)
 59 | func puncher(file *os.File, offset, size int64) error {
 60 | 	if err := ensureFileSparse(file); err != nil {
 61 | 		return err
 62 | 	}
 63 | 
 64 | 	// http://msdn.microsoft.com/en-us/library/windows/desktop/aa364411%28v=vs.85%29.aspx
 65 | 	// typedef struct _FILE_ZERO_DATA_INFORMATION {
 66 | 	//  LARGE_INTEGER FileOffset;
 67 | 	//  LARGE_INTEGER BeyondFinalZero;
 68 | 	//} FILE_ZERO_DATA_INFORMATION, *PFILE_ZERO_DATA_INFORMATION;
 69 | 	type fileZeroDataInformation struct {
 70 | 		FileOffset, BeyondFinalZero int64
 71 | 	}
 72 | 
 73 | 	lpInBuffer := fileZeroDataInformation{
 74 | 		FileOffset:      offset,
 75 | 		BeyondFinalZero: offset + size}
 76 | 	return deviceIOControl(false, file.Fd(), uintptr(unsafe.Pointer(&lpInBuffer)), 16)
 77 | }
 78 | 
 79 | // // http://msdn.microsoft.com/en-us/library/windows/desktop/cc948908%28v=vs.85%29.aspx
 80 | // type fileSetSparseBuffer struct {
 81 | //	 SetSparse bool
 82 | // }
 83 | 
 84 | func ensureFileSparse(file *os.File) (err error) {
 85 | 	fd := file.Fd()
 86 | 	sparseFilesMu.Lock()
 87 | 	if _, ok := sparseFiles[fd]; ok {
 88 | 		sparseFilesMu.Unlock()
 89 | 		return nil
 90 | 	}
 91 | 
 92 | 	if err = deviceIOControl(true, fd, 0, 0); err == nil {
 93 | 		sparseFiles[fd] = struct{}{}
 94 | 	}
 95 | 	sparseFilesMu.Unlock()
 96 | 	return err
 97 | }
 98 | 
 99 | func deviceIOControl(setSparse bool, fd, inBuf, inBufLen uintptr) (err error) {
100 | 	const (
101 | 		//http://source.winehq.org/source/include/winnt.h#L4605
102 | 		file_read_data  = 1
103 | 		file_write_data = 2
104 | 
105 | 		// METHOD_BUFFERED	0
106 | 		method_buffered = 0
107 | 		// FILE_ANY_ACCESS   0
108 | 		file_any_access = 0
109 | 		// FILE_DEVICE_FILE_SYSTEM   0x00000009
110 | 		file_device_file_system = 0x00000009
111 | 		// FILE_SPECIAL_ACCESS   (FILE_ANY_ACCESS)
112 | 		file_special_access = file_any_access
113 | 		file_read_access    = file_read_data
114 | 		file_write_access   = file_write_data
115 | 
116 | 		// http://source.winehq.org/source/include/winioctl.h
117 | 		// #define CTL_CODE 	(  	DeviceType,
118 | 		//		Function,
119 | 		//		Method,
120 | 		//		Access  		 )
121 | 		//    ((DeviceType) << 16) | ((Access) << 14) | ((Function) << 2) | (Method)
122 | 
123 | 		// FSCTL_SET_COMPRESSION   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 16, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA)
124 | 		fsctl_set_compression = (file_device_file_system << 16) | ((file_read_access | file_write_access) << 14) | (16 << 2) | method_buffered
125 | 		// FSCTL_SET_SPARSE   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 49, METHOD_BUFFERED, FILE_SPECIAL_ACCESS)
126 | 		fsctl_set_sparse = (file_device_file_system << 16) | (file_special_access << 14) | (49 << 2) | method_buffered
127 | 		// FSCTL_SET_ZERO_DATA   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 50, METHOD_BUFFERED, FILE_WRITE_DATA)
128 | 		fsctl_set_zero_data = (file_device_file_system << 16) | (file_write_data << 14) | (50 << 2) | method_buffered
129 | 	)
130 | 	retPtr := uintptr(unsafe.Pointer(&(make([]byte, 8)[0])))
131 | 	var r1 uintptr
132 | 	var e1 syscall.Errno
133 | 	if setSparse {
134 | 		// BOOL
135 | 		// WINAPI
136 | 		// DeviceIoControl( (HANDLE) hDevice,                      // handle to a file
137 | 		//                  FSCTL_SET_SPARSE,                      // dwIoControlCode
138 | 		//                  (PFILE_SET_SPARSE_BUFFER) lpInBuffer,  // input buffer
139 | 		//                  (DWORD) nInBufferSize,                 // size of input buffer
140 | 		//                  NULL,                                  // lpOutBuffer
141 | 		//                  0,                                     // nOutBufferSize
142 | 		//                  (LPDWORD) lpBytesReturned,             // number of bytes returned
143 | 		//                  (LPOVERLAPPED) lpOverlapped );         // OVERLAPPED structure
144 | 		r1, _, e1 = syscall.Syscall9(procDeviceIOControl.Addr(), 8,
145 | 			fd,
146 | 			uintptr(fsctl_set_sparse),
147 | 			// If the lpInBuffer parameter is NULL, the operation will behave the same as if the SetSparse member of the FILE_SET_SPARSE_BUFFER structure were TRUE. In other words, the operation sets the file to a sparse file.
148 | 			0, // uintptr(unsafe.Pointer(&lpInBuffer)),
149 | 			0, // 1,
150 | 			0,
151 | 			0,
152 | 			retPtr,
153 | 			0,
154 | 			0)
155 | 	} else {
156 | 		// BOOL
157 | 		// WINAPI
158 | 		// DeviceIoControl( (HANDLE) hDevice,              // handle to a file
159 | 		//                  FSCTL_SET_ZERO_DATA,           // dwIoControlCode
160 | 		//                  (LPVOID) lpInBuffer,           // input buffer
161 | 		//                  (DWORD) nInBufferSize,         // size of input buffer
162 | 		//                  NULL,                          // lpOutBuffer
163 | 		//                  0,                             // nOutBufferSize
164 | 		//                  (LPDWORD) lpBytesReturned,     // number of bytes returned
165 | 		//                  (LPOVERLAPPED) lpOverlapped ); // OVERLAPPED structure
166 | 		r1, _, e1 = syscall.Syscall9(procDeviceIOControl.Addr(), 8,
167 | 			fd,
168 | 			uintptr(fsctl_set_zero_data),
169 | 			inBuf,
170 | 			inBufLen,
171 | 			0,
172 | 			0,
173 | 			retPtr,
174 | 			0,
175 | 			0)
176 | 	}
177 | 	if r1 == 0 {
178 | 		if e1 != 0 {
179 | 			err = error(e1)
180 | 		} else {
181 | 			err = syscall.EINVAL
182 | 		}
183 | 	}
184 | 	return err
185 | }
186 | 


--------------------------------------------------------------------------------
/hdb/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | 
 3 | Redistribution and use in source and binary forms, with or without
 4 | modification, are permitted provided that the following conditions are
 5 | met:
 6 | 
 7 |    * Redistributions of source code must retain the above copyright
 8 | notice, this list of conditions and the following disclaimer.
 9 |    * Redistributions in binary form must reproduce the above
10 | copyright notice, this list of conditions and the following disclaimer
11 | in the documentation and/or other materials provided with the
12 | distribution.
13 |    * Neither the name of CZ.NIC nor the names of its
14 | contributors may be used to endorse or promote products derived from
15 | this software without specific prior written permission.
16 | 
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/hdb/README:
--------------------------------------------------------------------------------
1 | This is a goinstall-able mirror of modified code already published at:
2 | https://git.nic.cz/redmine/projects/gofileutil/repository/show/hdb
3 | 
4 | Install: $go get github.com/cznic/fileutil/hdb
5 | Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/hdb
6 | 


--------------------------------------------------------------------------------
/hdb/all_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package hdb
 8 | 
 9 | import (
10 | 	"testing"
11 | )
12 | 
13 | func TestPlaceholder(t *testing.T) {
14 | 	t.Log("TODO") //TODO
15 | }
16 | 


--------------------------------------------------------------------------------
/hdb/hdb.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // blame: jnml, labs.nic.cz
  6 | 
  7 | /*
  8 | WIP: Package hdb provides a "handle"/value DB like store, but actually it's
  9 | closer to the model of a process's virtual memory and its alloc, free and move
 10 | methods.
 11 | 
 12 | The hdb package is a thin layer around falloc.File providing stable-only
 13 | handles and the basic synchronizing primitives.  The central functionality of
 14 | hdb are the New, Set, Get and Delete methods of Store.
 15 | 
 16 | Conceptual analogy:
 17 | 	New	    alloc(sizeof(content)), return new "memory" pointer (a handle).
 18 | 
 19 | 	Get	    memmove() from "memory" "pointed to" by handle to the result content.
 20 | 		    Note: Handle "knows" the size of its content.
 21 | 
 22 | 	Set	    memmove() from content to "memory" pointed to by handle.
 23 | 		    In contrast to real memory, the new content may have different
 24 | 		    size than the previously stored one w/o additional handling
 25 | 		    and the "pointer" handle remains the same.
 26 | 
 27 | 	Delete	    free() the "memory" "pointed to" by handle.
 28 | */
 29 | package hdb
 30 | 
 31 | import (
 32 | 	"github.com/cznic/fileutil/falloc"
 33 | 	"github.com/cznic/fileutil/storage"
 34 | )
 35 | 
 36 | type Store struct {
 37 | 	f *falloc.File
 38 | }
 39 | 
 40 | // New returns a newly created Store backed by accessor, discarding its conents if any.
 41 | // If successful, methods on the returned Store can be used for I/O.
 42 | // It returns the Store and an error, if any.
 43 | func New(accessor storage.Accessor) (store *Store, err error) {
 44 | 	s := &Store{}
 45 | 	if s.f, err = falloc.New(accessor); err == nil {
 46 | 		store = s
 47 | 	}
 48 | 	return
 49 | }
 50 | 
 51 | // Open opens the Store from accessor.
 52 | // If successful, methods on the returned Store can be used for data exchange.
 53 | // It returns the Store and an error, if any.
 54 | func Open(accessor storage.Accessor) (store *Store, err error) {
 55 | 	s := &Store{}
 56 | 	if s.f, err = falloc.Open(accessor); err == nil {
 57 | 		store = s
 58 | 	}
 59 | 	return
 60 | }
 61 | 
 62 | // Close closes the store. Further access to the store has undefined behavior and may panic.
 63 | // It returns an error, if any.
 64 | func (s *Store) Close() (err error) {
 65 | 	defer func() {
 66 | 		s.f = nil
 67 | 	}()
 68 | 
 69 | 	return s.f.Close()
 70 | }
 71 | 
 72 | // Delete deletes the data associated with handle.
 73 | // It returns an error if any.
 74 | func (s *Store) Delete(handle falloc.Handle) (err error) {
 75 | 	return s.f.Free(handle)
 76 | }
 77 | 
 78 | // Get gets the data associated with handle.
 79 | // It returns the data and an error, if any.
 80 | func (s *Store) Get(handle falloc.Handle) (b []byte, err error) {
 81 | 	return s.f.Read(handle)
 82 | }
 83 | 
 84 | // New associates data with a new handle.
 85 | // It returns the handle and an error, if any.
 86 | func (s *Store) New(b []byte) (handle falloc.Handle, err error) {
 87 | 	return s.f.Alloc(b)
 88 | }
 89 | 
 90 | // Set associates data with an existing handle.
 91 | // It returns an error, if any.
 92 | func (s *Store) Set(handle falloc.Handle, b []byte) (err error) {
 93 | 	_, err = s.f.Realloc(handle, b, true)
 94 | 	return
 95 | }
 96 | 
 97 | // Root returns the handle of the DB root (top level directory, ...).
 98 | func (s *Store) Root() falloc.Handle {
 99 | 	return s.f.Root()
100 | }
101 | 
102 | // File returns the underlying falloc.File of 's'.
103 | func (s *Store) File() *falloc.File {
104 | 	return s.f
105 | }
106 | 
107 | // Lock locks 's' for writing. If the lock is already locked for reading or writing,
108 | // Lock blocks until the lock is available. To ensure that the lock eventually becomes available,
109 | // a blocked Lock call excludes new readers from acquiring the lock.
110 | func (s *Store) Lock() {
111 | 	s.f.Lock()
112 | }
113 | 
114 | // RLock locks 's' for reading. If the lock is already locked for writing or there is a writer
115 | // already waiting to release the lock, RLock blocks until the writer has released the lock.
116 | func (s *Store) RLock() {
117 | 	s.f.RLock()
118 | }
119 | 
120 | // Unlock unlocks 's' for writing. It's a run-time error if 's' is not locked for writing on entry to Unlock.
121 | //
122 | // As with Mutexes, a locked RWMutex is not associated with a particular goroutine.
123 | // One goroutine may RLock (Lock) 's' and then arrange for another goroutine to RUnlock (Unlock) it.
124 | func (s *Store) Unlock() {
125 | 	s.f.Unlock()
126 | }
127 | 
128 | // RUnlock undoes a single RLock call; it does not affect other simultaneous readers.
129 | // It's a run-time error if 's' is not locked for reading on entry to RUnlock.
130 | func (s *Store) RUnlock() {
131 | 	s.f.RUnlock()
132 | }
133 | 
134 | // LockedNew wraps New in a Lock/Unlock pair.
135 | func (s *Store) LockedNew(b []byte) (handle falloc.Handle, err error) {
136 | 	return s.f.LockedAlloc(b)
137 | }
138 | 
139 | // LockedDelete wraps Delete in a Lock/Unlock pair.
140 | func (s *Store) LockedDelete(handle falloc.Handle) (err error) {
141 | 	return s.f.LockedFree(handle)
142 | }
143 | 
144 | // LockedGet wraps Get in a RLock/RUnlock pair.
145 | func (s *Store) LockedGet(handle falloc.Handle) (b []byte, err error) {
146 | 	return s.f.LockedRead(handle)
147 | }
148 | 
149 | // LockedSet wraps Set in a Lock/Unlock pair.
150 | func (s *Store) LockedSet(handle falloc.Handle, b []byte) (err error) {
151 | 	_, err = s.f.Realloc(handle, b, true)
152 | 	return
153 | }
154 | 


--------------------------------------------------------------------------------
/hdb/test_deps.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package hdb
 8 | 
 9 | // Pull test dependencies too.
10 | // Enables easy 'go test X' after 'go get X'
11 | import (
12 | // nothing yet
13 | )
14 | 


--------------------------------------------------------------------------------
/punch_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // +build amd64 amd64p32 arm64 arm64be ppc64 ppc64le mips64 mips64le mips64p32 mips64p32le sparc64
 6 | 
 7 | package fileutil
 8 | 
 9 | import (
10 | 	"io/ioutil"
11 | 	"os"
12 | 	"testing"
13 | )
14 | 
15 | func TestPunch(t *testing.T) {
16 | 	file, err := ioutil.TempFile("", "punchhole-")
17 | 	if err != nil {
18 | 		t.Error(err)
19 | 	}
20 | 	defer os.Remove(file.Name())
21 | 	defer file.Close()
22 | 	buf := make([]byte, 10<<20)
23 | 	for i := range buf {
24 | 		buf[i] = byte(1 + (i+1)&0xfe)
25 | 	}
26 | 	if _, err = file.Write(buf); err != nil {
27 | 		t.Errorf("error writing to the temp file: %v", err)
28 | 		t.FailNow()
29 | 	}
30 | 	if err = file.Sync(); err != nil {
31 | 		t.Logf("error syncing %q: %v", file.Name(), err)
32 | 	}
33 | 	for i, j := range []int{1, 31, 1 << 10} {
34 | 		if err = PunchHole(file, int64(j), int64(j)); err != nil {
35 | 			t.Errorf("%d. error punching at %d, size %d: %v", i, j, j, err)
36 | 			continue
37 | 		}
38 | 		// read back, with 1-1 bytes overlaid
39 | 		n, err := file.ReadAt(buf[:j+2], int64(j-1))
40 | 		if err != nil {
41 | 			t.Errorf("%d. error reading file: %v", i, err)
42 | 			continue
43 | 		}
44 | 		buf = buf[:n]
45 | 		if buf[0] == 0 {
46 | 			t.Errorf("%d. file at %d has been overwritten with 0!", i, j-1)
47 | 		}
48 | 		if buf[n-1] == 0 {
49 | 			t.Errorf("%d. file at %d has been overwritten with 0!", i, j-1+n)
50 | 		}
51 | 		if !hasPunchHole {
52 | 			continue
53 | 		}
54 | 
55 | 		for k, v := range buf[1 : n-1] {
56 | 			if v != 0 {
57 | 				t.Errorf("%d. error reading file at %d got %d, want 0.", i, k, v)
58 | 			}
59 | 		}
60 | 	}
61 | }
62 | 


--------------------------------------------------------------------------------
/storage/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | 
 3 | Redistribution and use in source and binary forms, with or without
 4 | modification, are permitted provided that the following conditions are
 5 | met:
 6 | 
 7 |    * Redistributions of source code must retain the above copyright
 8 | notice, this list of conditions and the following disclaimer.
 9 |    * Redistributions in binary form must reproduce the above
10 | copyright notice, this list of conditions and the following disclaimer
11 | in the documentation and/or other materials provided with the
12 | distribution.
13 |    * Neither the name of CZ.NIC nor the names of its
14 | contributors may be used to endorse or promote products derived from
15 | this software without specific prior written permission.
16 | 
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/storage/README:
--------------------------------------------------------------------------------
1 | This is a goinstall-able mirror of modified code already published at:
2 | https://git.nic.cz/redmine/projects/gofileutil/repository/show/storage
3 | 
4 | Install: $go get github.com/cznic/fileutil/storage
5 | Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/storage
6 | 


--------------------------------------------------------------------------------
/storage/all_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | import (
10 | 	"flag"
11 | 	"runtime"
12 | )
13 | 
14 | var (
15 | 	devFlag = flag.Bool("dev", false, "enable dev tests")
16 | 	goFlag  = flag.Int("go", 1, "GOMAXPROCS")
17 | )
18 | 
19 | func init() {
20 | 	flag.Parse()
21 | 	runtime.GOMAXPROCS(*goFlag)
22 | }
23 | 


--------------------------------------------------------------------------------
/storage/cache.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // blame: jnml, labs.nic.cz
  6 | 
  7 | package storage
  8 | 
  9 | import (
 10 | 	"container/list"
 11 | 	"io"
 12 | 	"math"
 13 | 	"os"
 14 | 	"sync"
 15 | 	"sync/atomic"
 16 | )
 17 | 
 18 | type cachepage struct {
 19 | 	b     [512]byte
 20 | 	dirty bool
 21 | 	lru   *list.Element
 22 | 	pi    int64
 23 | 	valid int // page content is b[:valid]
 24 | }
 25 | 
 26 | func (p *cachepage) wr(b []byte, off int) (wasDirty bool) {
 27 | 	copy(p.b[off:], b)
 28 | 	if n := off + len(b); n > p.valid {
 29 | 		p.valid = n
 30 | 	}
 31 | 	wasDirty = p.dirty
 32 | 	p.dirty = true
 33 | 	return
 34 | }
 35 | 
 36 | func (c *Cache) rd(off int64, read bool) (p *cachepage, ok bool) {
 37 | 	c.Rq++
 38 | 	pi := off >> 9
 39 | 	if p, ok = c.m[pi]; ok {
 40 | 		c.lru.MoveToBack(p.lru)
 41 | 		return
 42 | 	}
 43 | 
 44 | 	if !read {
 45 | 		return
 46 | 	}
 47 | 
 48 | 	fp := off &^ 511
 49 | 	if fp >= c.size {
 50 | 		return
 51 | 	}
 52 | 
 53 | 	rq := 512
 54 | 	if fp+512 > c.size {
 55 | 		rq = int(c.size - fp)
 56 | 	}
 57 | 	p = &cachepage{pi: pi, valid: rq}
 58 | 	p.lru = c.lru.PushBack(p)
 59 | 	if n, err := c.f.ReadAt(p.b[:p.valid], fp); n != rq {
 60 | 		panic(err)
 61 | 	}
 62 | 
 63 | 	c.Load++
 64 | 	if c.advise != nil {
 65 | 		c.advise(fp, 512, false)
 66 | 	}
 67 | 	c.m[pi], ok = p, true
 68 | 	return
 69 | }
 70 | 
 71 | func (c *Cache) wr(off int64) (p *cachepage) {
 72 | 	var ok bool
 73 | 	if p, ok = c.rd(off, false); ok {
 74 | 		return
 75 | 	}
 76 | 
 77 | 	pi := off >> 9
 78 | 	p = &cachepage{pi: pi}
 79 | 	p.lru = c.lru.PushBack(p)
 80 | 	c.m[pi] = p
 81 | 	return
 82 | }
 83 | 
 84 | // Cache provides caching support for another store Accessor.
 85 | type Cache struct {
 86 | 	advise   func(int64, int, bool)
 87 | 	clean    chan bool
 88 | 	cleaning int32
 89 | 	close    chan bool
 90 | 	f        Accessor
 91 | 	fi       *FileInfo
 92 | 	lock     sync.Mutex
 93 | 	lru      *list.List
 94 | 	m        map[int64]*cachepage
 95 | 	maxpages int
 96 | 	size     int64
 97 | 	sync     chan bool
 98 | 	wlist    *list.List
 99 | 	write    chan bool
100 | 	writing  int32
101 | 	Rq       int64 // Pages requested from cache
102 | 	Load     int64 // Pages loaded (cache miss)
103 | 	Purge    int64 // Pages purged
104 | 	Top      int   // "High water" pages
105 | }
106 | 
107 | // Implementation of Accessor.
108 | func (c *Cache) BeginUpdate() error { return nil }
109 | 
110 | // Implementation of Accessor.
111 | func (c *Cache) EndUpdate() error { return nil }
112 | 
113 | // NewCache creates a caching Accessor from store with total of maxcache bytes.
114 | // NewCache returns the new Cache, implementing Accessor or an error if any.
115 | //
116 | // The LRU mechanism is used, so the cache tries to keep often accessed pages cached.
117 | //
118 | func NewCache(store Accessor, maxcache int64, advise func(int64, int, bool)) (c *Cache, err error) {
119 | 	var fi os.FileInfo
120 | 	if fi, err = store.Stat(); err != nil {
121 | 		return
122 | 	}
123 | 
124 | 	x := maxcache >> 9
125 | 	if x > math.MaxInt32/2 {
126 | 		x = math.MaxInt32 / 2
127 | 	}
128 | 	c = &Cache{
129 | 		advise:   advise,
130 | 		clean:    make(chan bool, 1),
131 | 		close:    make(chan bool),
132 | 		f:        store,
133 | 		lru:      list.New(), // front == oldest used, back == last recently used
134 | 		m:        make(map[int64]*cachepage),
135 | 		maxpages: int(x),
136 | 		size:     fi.Size(),
137 | 		sync:     make(chan bool),
138 | 		wlist:    list.New(),
139 | 		write:    make(chan bool, 1),
140 | 	}
141 | 	c.fi = NewFileInfo(fi, c)
142 | 	go c.writer()
143 | 	go c.cleaner(int((int64(c.maxpages) * 95) / 100)) // hysteresis
144 | 	return
145 | }
146 | 
147 | func (c *Cache) Accessor() Accessor {
148 | 	return c.f
149 | }
150 | 
151 | func (c *Cache) Close() (err error) {
152 | 	close(c.write)
153 | 	<-c.close
154 | 	close(c.clean)
155 | 	<-c.close
156 | 	return c.f.Close()
157 | }
158 | 
159 | func (c *Cache) Name() (s string) {
160 | 	return c.f.Name()
161 | }
162 | 
163 | func (c *Cache) ReadAt(b []byte, off int64) (n int, err error) {
164 | 	po := int(off) & 0x1ff
165 | 	bp := 0
166 | 	rem := len(b)
167 | 	m := 0
168 | 	for rem != 0 {
169 | 		c.lock.Lock() // X1+
170 | 		p, ok := c.rd(off, true)
171 | 		if !ok {
172 | 			c.lock.Unlock() // X1-
173 | 			return -1, io.EOF
174 | 		}
175 | 
176 | 		rq := rem
177 | 		if po+rq > 512 {
178 | 			rq = 512 - po
179 | 		}
180 | 		if n := copy(b[bp:bp+rq], p.b[po:p.valid]); n != rq {
181 | 			c.lock.Unlock() // X1-
182 | 			return -1, io.EOF
183 | 		}
184 | 
185 | 		m = len(c.m)
186 | 		c.lock.Unlock() // X1-
187 | 		po = 0
188 | 		bp += rq
189 | 		off += int64(rq)
190 | 		rem -= rq
191 | 		n += rq
192 | 	}
193 | 	if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) {
194 | 		if m > c.Top {
195 | 			c.Top = m
196 | 		}
197 | 		c.clean <- true
198 | 	}
199 | 	return
200 | }
201 | 
202 | func (c *Cache) Stat() (fi os.FileInfo, err error) {
203 | 	c.lock.Lock()
204 | 	defer c.lock.Unlock()
205 | 	return c.fi, nil
206 | }
207 | 
208 | func (c *Cache) Sync() (err error) {
209 | 	c.write <- false
210 | 	<-c.sync
211 | 	return
212 | }
213 | 
214 | func (c *Cache) Truncate(size int64) (err error) {
215 | 	c.Sync() //TODO improve (discard pages, the writer goroutine should also be aware, ...)
216 | 	c.lock.Lock()
217 | 	defer c.lock.Unlock()
218 | 	c.size = size
219 | 	return c.f.Truncate(size)
220 | }
221 | 
222 | func (c *Cache) WriteAt(b []byte, off int64) (n int, err error) {
223 | 	po := int(off) & 0x1ff
224 | 	bp := 0
225 | 	rem := len(b)
226 | 	m := 0
227 | 	for rem != 0 {
228 | 		c.lock.Lock() // X+
229 | 		p := c.wr(off)
230 | 		rq := rem
231 | 		if po+rq > 512 {
232 | 			rq = 512 - po
233 | 		}
234 | 		if wasDirty := p.wr(b[bp:bp+rq], po); !wasDirty {
235 | 			c.wlist.PushBack(p)
236 | 		}
237 | 		m = len(c.m)
238 | 		po = 0
239 | 		bp += rq
240 | 		off += int64(rq)
241 | 		if off > c.size {
242 | 			c.size = off
243 | 		}
244 | 		c.lock.Unlock() // X-
245 | 		rem -= rq
246 | 		n += rq
247 | 	}
248 | 	if atomic.CompareAndSwapInt32(&c.writing, 0, 1) {
249 | 		c.write <- true
250 | 	}
251 | 	if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) {
252 | 		if m > c.Top {
253 | 			c.Top = m
254 | 		}
255 | 		c.clean <- true
256 | 	}
257 | 	return
258 | }
259 | 
260 | func (c *Cache) writer() {
261 | 	for ok := true; ok; {
262 | 		var wr bool
263 | 		var off int64
264 | 		wr, ok = <-c.write
265 | 		for {
266 | 			c.lock.Lock() // X1+
267 | 			item := c.wlist.Front()
268 | 			if item == nil {
269 | 				c.lock.Unlock() // X1-
270 | 				break
271 | 			}
272 | 
273 | 			p := item.Value.(*cachepage)
274 | 			off = p.pi << 9
275 | 			if n, err := c.f.WriteAt(p.b[:p.valid], off); n != p.valid {
276 | 				c.lock.Unlock()                    // X1-
277 | 				panic("TODO Cache.writer errchan") //TODO +errchan
278 | 				panic(err)
279 | 			}
280 | 
281 | 			p.dirty = false
282 | 			c.wlist.Remove(item)
283 | 			if c.advise != nil {
284 | 				c.advise(off, 512, true)
285 | 			}
286 | 			c.lock.Unlock() // X1-
287 | 		}
288 | 		switch {
289 | 		case wr:
290 | 			atomic.AddInt32(&c.writing, -1)
291 | 		case ok:
292 | 			c.sync <- true
293 | 		}
294 | 	}
295 | 	c.close <- true
296 | }
297 | 
298 | func (c *Cache) cleaner(limit int) {
299 | 	for _ = range c.clean {
300 | 		var item *list.Element
301 | 		for {
302 | 			c.lock.Lock() // X1+
303 | 			if len(c.m) < limit {
304 | 				c.lock.Unlock() // X1-
305 | 				break
306 | 			}
307 | 
308 | 			if item == nil {
309 | 				item = c.lru.Front()
310 | 			}
311 | 			if p := item.Value.(*cachepage); !p.dirty {
312 | 				delete(c.m, p.pi)
313 | 				c.lru.Remove(item)
314 | 				c.Purge++
315 | 			}
316 | 			item = item.Next()
317 | 			c.lock.Unlock() // X1-
318 | 		}
319 | 		atomic.AddInt32(&c.cleaning, -1)
320 | 	}
321 | 	c.close <- true
322 | }
323 | 


--------------------------------------------------------------------------------
/storage/cache_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | import (
10 | 	"io/ioutil"
11 | 	"os"
12 | 	"path/filepath"
13 | 	"testing"
14 | )
15 | 
16 | func newfile(t *testing.T) (string, string, Accessor) {
17 | 	dir, err := ioutil.TempDir("", "test-storage-")
18 | 	if err != nil {
19 | 		panic(err)
20 | 	}
21 | 
22 | 	name := filepath.Join(dir, "test.tmp")
23 | 	f, err := NewFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666)
24 | 	if err != nil {
25 | 		t.Fatal("newfile", err)
26 | 	}
27 | 
28 | 	return dir, name, f
29 | }
30 | 
31 | func readfile(t *testing.T, name string) (b []byte) {
32 | 	var err error
33 | 	if b, err = ioutil.ReadFile(name); err != nil {
34 | 		t.Fatal("readfile")
35 | 	}
36 | 
37 | 	return
38 | }
39 | 
40 | func newcache(t *testing.T) (dir, name string, c *Cache) {
41 | 	dir, name, f := newfile(t)
42 | 	var err error
43 | 	if c, err = NewCache(f, 1<<20, nil); err != nil {
44 | 		t.Fatal("newCache", err)
45 | 	}
46 | 
47 | 	return
48 | }
49 | 
50 | func TestCache0(t *testing.T) {
51 | 	dir, name, c := newcache(t)
52 | 	defer os.RemoveAll(dir)
53 | 
54 | 	if err := c.Close(); err != nil {
55 | 		t.Fatal(10, err)
56 | 	}
57 | 
58 | 	if b := readfile(t, name); len(b) != 0 {
59 | 		t.Fatal(20, len(b), 0)
60 | 	}
61 | }
62 | 
63 | func TestCache1(t *testing.T) {
64 | 	dir, name, c := newcache(t)
65 | 	defer os.RemoveAll(dir)
66 | 
67 | 	if n, err := c.WriteAt([]byte{0xa5}, 0); n != 1 {
68 | 		t.Fatal(20, n, err)
69 | 	}
70 | 
71 | 	if err := c.Close(); err != nil {
72 | 		t.Fatal(10, err)
73 | 	}
74 | 
75 | 	b := readfile(t, name)
76 | 	if len(b) != 1 {
77 | 		t.Fatal(30, len(b), 1)
78 | 	}
79 | 
80 | 	if b[0] != 0xa5 {
81 | 		t.Fatal(40, b[0], 0xa5)
82 | 	}
83 | }
84 | 


--------------------------------------------------------------------------------
/storage/dev_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | import (
10 | 	"testing"
11 | )
12 | 
13 | func TestDevNothing(t *testing.T) {
14 | 	if !*devFlag {
15 | 		t.Log("not enabled")
16 | 		return
17 | 	}
18 | }
19 | 


--------------------------------------------------------------------------------
/storage/file.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | import (
10 | 	"os"
11 | )
12 | 
13 | // FileAccessor is the concrete type returned by NewFile and OpenFile.
14 | type FileAccessor struct {
15 | 	*os.File
16 | }
17 | 
18 | // Implementation of Accessor.
19 | func (f *FileAccessor) BeginUpdate() error { return nil }
20 | 
21 | // Implementation of Accessor.
22 | func (f *FileAccessor) EndUpdate() error { return nil }
23 | 
24 | // NewFile returns an Accessor backed by an os.File named name, It opens the
25 | // named file with specified flag (os.O_RDWR etc.) and perm, (0666 etc.) if
26 | // applicable.  If successful, methods on the returned Accessor can be used for
27 | // I/O.  It returns the Accessor and an Error, if any.
28 | //
29 | // NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
30 | func NewFile(name string, flag int, perm os.FileMode) (store Accessor, err error) {
31 | 	var f FileAccessor
32 | 	if f.File, err = os.OpenFile(name, flag, perm); err == nil {
33 | 		store = &f
34 | 	}
35 | 	return
36 | }
37 | 
38 | // OpenFile returns an Accessor backed by an existing os.File named name, It
39 | // opens the named file with specified flag (os.O_RDWR etc.) and perm, (0666
40 | // etc.) if applicable.  If successful, methods on the returned Accessor can be
41 | // used for I/O.  It returns the Accessor and an Error, if any.
42 | //
43 | // NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
44 | func OpenFile(name string, flag int, perm os.FileMode) (store Accessor, err error) {
45 | 	var f FileAccessor
46 | 	if f.File, err = os.OpenFile(name, flag, perm); err == nil {
47 | 		store = &f
48 | 	}
49 | 	return
50 | }
51 | 


--------------------------------------------------------------------------------
/storage/mem.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // blame: jnml, labs.nic.cz
  6 | 
  7 | package storage
  8 | 
  9 | import (
 10 | 	"errors"
 11 | 	"fmt"
 12 | 	"io/ioutil"
 13 | 	"math"
 14 | 	"os"
 15 | )
 16 | 
 17 | //TODO -> exported type w/ exported fields
 18 | type memaccessor struct {
 19 | 	f  *os.File
 20 | 	fi *FileInfo
 21 | 	b  []byte
 22 | }
 23 | 
 24 | // Implementation of Accessor.
 25 | func (m *memaccessor) BeginUpdate() error { return nil }
 26 | 
 27 | // Implementation of Accessor.
 28 | func (f *memaccessor) EndUpdate() error { return nil }
 29 | 
 30 | // NewMem returns a new Accessor backed by an os.File.  The returned Accessor
 31 | // keeps all of the store content in memory.  The memory and file images are
 32 | // synced only by Sync and Close.  Recomended for small amounts of data only
 33 | // and content which may be lost on process kill/crash.  NewMem return the
 34 | // Accessor or an error of any.
 35 | //
 36 | // NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
 37 | func NewMem(f *os.File) (store Accessor, err error) {
 38 | 	a := &memaccessor{f: f}
 39 | 	if err = f.Truncate(0); err != nil {
 40 | 		return
 41 | 	}
 42 | 
 43 | 	var fi os.FileInfo
 44 | 	if fi, err = a.f.Stat(); err != nil {
 45 | 		return
 46 | 	}
 47 | 
 48 | 	a.fi = NewFileInfo(fi, a)
 49 | 	store = a
 50 | 	return
 51 | }
 52 | 
 53 | // OpenMem return a new Accessor backed by an os.File.  The store content is
 54 | // loaded from f.  The returned Accessor keeps all of the store content in
 55 | // memory.  The memory and file images are synced only Sync and Close.
 56 | // Recomended for small amounts of data only and content which may be lost on
 57 | // process kill/crash.  OpenMem return the Accessor or an error of any.
 58 | //
 59 | // NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
 60 | func OpenMem(f *os.File) (store Accessor, err error) {
 61 | 	a := &memaccessor{f: f}
 62 | 	if a.b, err = ioutil.ReadAll(a.f); err != nil {
 63 | 		a.f.Close()
 64 | 		return
 65 | 	}
 66 | 
 67 | 	var fi os.FileInfo
 68 | 	if fi, err = a.f.Stat(); err != nil {
 69 | 		a.f.Close()
 70 | 		return
 71 | 	}
 72 | 
 73 | 	a.fi = NewFileInfo(fi, a)
 74 | 	store = a
 75 | 	return
 76 | }
 77 | 
 78 | // Close implements Accessor. Specifically it synchronizes the memory and file images.
 79 | func (a *memaccessor) Close() (err error) {
 80 | 	defer func() {
 81 | 		a.b = nil
 82 | 		if a.f != nil {
 83 | 			if e := a.f.Close(); e != nil && err == nil {
 84 | 				err = e
 85 | 			}
 86 | 		}
 87 | 		a.f = nil
 88 | 	}()
 89 | 
 90 | 	return a.Sync()
 91 | }
 92 | 
 93 | func (a *memaccessor) Name() string {
 94 | 	return a.f.Name()
 95 | }
 96 | 
 97 | func (a *memaccessor) ReadAt(b []byte, off int64) (n int, err error) {
 98 | 	if off < 0 || off > math.MaxInt32 {
 99 | 		return -1, fmt.Errorf("ReadAt: illegal offset %#x", off)
100 | 	}
101 | 
102 | 	rq, fp := len(b), int(off)
103 | 	if fp+rq > len(a.b) {
104 | 		return -1, fmt.Errorf("ReadAt: illegal rq %#x @ offset %#x, len %#x", rq, fp, len(a.b))
105 | 	}
106 | 
107 | 	copy(b, a.b[fp:])
108 | 	return
109 | }
110 | 
111 | func (a *memaccessor) Stat() (fi os.FileInfo, err error) {
112 | 	i := a.fi
113 | 	i.FSize = int64(len(a.b))
114 | 	fi = i
115 | 	return
116 | }
117 | 
118 | // Sync implements Accessor. Specifically it synchronizes the memory and file images.
119 | func (a *memaccessor) Sync() (err error) {
120 | 	var n int
121 | 	if n, err = a.f.WriteAt(a.b, 0); n != len(a.b) {
122 | 		return
123 | 	}
124 | 
125 | 	return a.f.Truncate(int64(len(a.b)))
126 | }
127 | 
128 | func (a *memaccessor) Truncate(size int64) (err error) {
129 | 	defer func() {
130 | 		if e := recover(); e != nil {
131 | 			err = e.(error)
132 | 		}
133 | 	}()
134 | 
135 | 	if size > math.MaxInt32 {
136 | 		panic(errors.New("truncate: illegal size"))
137 | 	}
138 | 
139 | 	a.b = a.b[:int(size)]
140 | 	return
141 | }
142 | 
143 | func (a *memaccessor) WriteAt(b []byte, off int64) (n int, err error) {
144 | 	if off < 0 || off > math.MaxInt32 {
145 | 		return -1, errors.New("WriteAt: illegal offset")
146 | 	}
147 | 
148 | 	rq, fp, size := len(b), int(off), len(a.b)
149 | 	if need := rq + fp; need > size {
150 | 		if need <= cap(a.b) {
151 | 			a.b = a.b[:need]
152 | 		} else {
153 | 			nb := make([]byte, need, 2*need)
154 | 			copy(nb, a.b)
155 | 			a.b = nb
156 | 		}
157 | 	}
158 | 
159 | 	copy(a.b[int(off):], b)
160 | 	return
161 | }
162 | 


--------------------------------------------------------------------------------
/storage/mem_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | import (
10 | 	"testing"
11 | )
12 | 
13 | func Test(t *testing.T) {
14 | 	t.Log("TODO placeholder") //TODO
15 | }
16 | 


--------------------------------------------------------------------------------
/storage/probe.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | import "sync/atomic"
10 | 
11 | // Probe collects usage statistics of the embeded Accessor.
12 | // Probe itself IS an Accessor.
13 | type Probe struct {
14 | 	Accessor
15 | 	Chain     *Probe
16 | 	OpsRd     int64
17 | 	OpsWr     int64
18 | 	BytesRd   int64
19 | 	BytesWr   int64
20 | 	SectorsRd int64 // Assuming 512 byte sector size
21 | 	SectorsWr int64
22 | }
23 | 
24 | // NewProbe returns a newly created probe which embedes the src Accessor.
25 | // The retuned *Probe satisfies Accessor. if chain != nil then Reset()
26 | // is cascaded down the chained Probes.
27 | func NewProbe(src Accessor, chain *Probe) *Probe {
28 | 	return &Probe{Accessor: src, Chain: chain}
29 | }
30 | 
31 | func reset(n *int64) {
32 | 	atomic.AddInt64(n, -atomic.AddInt64(n, 0))
33 | }
34 | 
35 | // Reset zeroes the collected statistics of p.
36 | func (p *Probe) Reset() {
37 | 	if p.Chain != nil {
38 | 		p.Chain.Reset()
39 | 	}
40 | 	reset(&p.OpsRd)
41 | 	reset(&p.OpsWr)
42 | 	reset(&p.BytesRd)
43 | 	reset(&p.BytesWr)
44 | 	reset(&p.SectorsRd)
45 | 	reset(&p.SectorsWr)
46 | }
47 | 
48 | func (p *Probe) ReadAt(b []byte, off int64) (n int, err error) {
49 | 	n, err = p.Accessor.ReadAt(b, off)
50 | 	atomic.AddInt64(&p.OpsRd, 1)
51 | 	atomic.AddInt64(&p.BytesRd, int64(n))
52 | 	if n <= 0 {
53 | 		return
54 | 	}
55 | 
56 | 	sectorFirst := off >> 9
57 | 	sectorLast := (off + int64(n) - 1) >> 9
58 | 	atomic.AddInt64(&p.SectorsRd, sectorLast-sectorFirst+1)
59 | 	return
60 | }
61 | 
62 | func (p *Probe) WriteAt(b []byte, off int64) (n int, err error) {
63 | 	n, err = p.Accessor.WriteAt(b, off)
64 | 	atomic.AddInt64(&p.OpsWr, 1)
65 | 	atomic.AddInt64(&p.BytesWr, int64(n))
66 | 	if n <= 0 {
67 | 		return
68 | 	}
69 | 
70 | 	sectorFirst := off >> 9
71 | 	sectorLast := (off + int64(n) - 1) >> 9
72 | 	atomic.AddInt64(&p.SectorsWr, sectorLast-sectorFirst+1)
73 | 	return
74 | }
75 | 


--------------------------------------------------------------------------------
/storage/probe_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | import (
10 | 	"os"
11 | 	"testing"
12 | )
13 | 
14 | func (p *Probe) assert(t *testing.T, msg int, opsRd, opsWr, bytesRd, bytesWr, sectorsRd, sectorsWr int64) {
15 | 	if n := p.OpsRd; n != opsRd {
16 | 		t.Fatal(msg, n, opsRd)
17 | 	}
18 | 
19 | 	if n := p.OpsWr; n != opsWr {
20 | 		t.Fatal(msg+1, n, opsWr)
21 | 	}
22 | 
23 | 	if n := p.BytesRd; n != bytesRd {
24 | 		t.Fatal(msg+2, n, bytesRd)
25 | 	}
26 | 
27 | 	if n := p.BytesWr; n != bytesWr {
28 | 		t.Fatal(msg+3, n, bytesWr)
29 | 	}
30 | 
31 | 	if n := p.SectorsRd; n != sectorsRd {
32 | 		t.Fatal(msg+4, n, sectorsRd)
33 | 	}
34 | 
35 | 	if n := p.SectorsWr; n != sectorsWr {
36 | 		t.Fatal(msg+5, n, sectorsWr)
37 | 	}
38 | }
39 | 
40 | func TestProbe(t *testing.T) {
41 | 	return //TODO disabled due to atomic.AddInt64 failing on W32
42 | 	const fn = "test.tmp"
43 | 
44 | 	store, err := NewFile(fn, os.O_CREATE|os.O_RDWR|os.O_CREATE, 0666)
45 | 	if err != nil {
46 | 		t.Fatal(10, err)
47 | 	}
48 | 
49 | 	defer func() {
50 | 		ec := store.Close()
51 | 		er := os.Remove(fn)
52 | 		if ec != nil {
53 | 			t.Fatal(10000, ec)
54 | 		}
55 | 		if er != nil {
56 | 			t.Fatal(10001, er)
57 | 		}
58 | 	}()
59 | 
60 | 	probe := NewProbe(store, nil)
61 | 	if n, err := probe.WriteAt([]byte{1}, 0); n != 1 {
62 | 		t.Fatal(20, err)
63 | 	}
64 | 
65 | 	probe.assert(t, 30, 0, 1, 0, 1, 0, 1)
66 | 	b := []byte{0}
67 | 	if n, err := probe.ReadAt(b, 0); n != 1 {
68 | 		t.Fatal(40, err)
69 | 	}
70 | 
71 | 	if n := b[0]; n != 1 {
72 | 		t.Fatal(50, n, 1)
73 | 	}
74 | 
75 | 	probe.assert(t, 60, 1, 1, 1, 1, 1, 1)
76 | 	if n, err := probe.WriteAt([]byte{2, 3}, 510); n != 2 {
77 | 		t.Fatal(70, err)
78 | 	}
79 | 
80 | 	probe.assert(t, 80, 1, 2, 1, 3, 1, 2)
81 | 	if n, err := probe.WriteAt([]byte{2, 3}, 511); n != 2 {
82 | 		t.Fatal(90, err)
83 | 	}
84 | 
85 | 	probe.assert(t, 100, 1, 3, 1, 5, 1, 4)
86 | }
87 | 


--------------------------------------------------------------------------------
/storage/storage.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | // blame: jnml, labs.nic.cz
  6 | 
  7 | // WIP: Package storage defines and implements storage providers and store accessors.
  8 | package storage
  9 | 
 10 | import (
 11 | 	"os"
 12 | 	"sync"
 13 | 	"time"
 14 | )
 15 | 
 16 | // FileInfo is a type implementing os.FileInfo which has setable fields, like
 17 | // the older os.FileInfo used to have. It is used wehere e.g. the Size is
 18 | // needed to be faked (encapsulated/memory only file, file cache, etc.).
 19 | type FileInfo struct {
 20 | 	FName    string      // base name of the file
 21 | 	FSize    int64       // length in bytes
 22 | 	FMode    os.FileMode // file mode bits
 23 | 	FModTime time.Time   // modification time
 24 | 	FIsDir   bool        // abbreviation for Mode().IsDir()
 25 | 	sys      interface{} // underlying data source (can be nil)
 26 | }
 27 | 
 28 | // NewFileInfo creates FileInfo from os.FileInfo fi.
 29 | func NewFileInfo(fi os.FileInfo, sys interface{}) *FileInfo {
 30 | 	return &FileInfo{fi.Name(), fi.Size(), fi.Mode(), fi.ModTime(), fi.IsDir(), sys}
 31 | }
 32 | 
 33 | // Implementation of os.FileInfo
 34 | func (fi *FileInfo) Name() string {
 35 | 	return fi.FName
 36 | }
 37 | 
 38 | // Implementation of os.FileInfo
 39 | func (fi *FileInfo) Size() int64 {
 40 | 	return fi.FSize
 41 | }
 42 | 
 43 | // Implementation of os.FileInfo
 44 | func (fi *FileInfo) Mode() os.FileMode {
 45 | 	return fi.FMode
 46 | }
 47 | 
 48 | // Implementation of os.FileInfo
 49 | func (fi *FileInfo) ModTime() time.Time {
 50 | 	return fi.FModTime
 51 | }
 52 | 
 53 | // Implementation of os.FileInfo
 54 | func (fi *FileInfo) IsDir() bool {
 55 | 	return fi.FIsDir
 56 | }
 57 | 
 58 | func (fi *FileInfo) Sys() interface{} {
 59 | 	return fi.sys
 60 | }
 61 | 
 62 | // Accessor provides I/O methods to access a store.
 63 | type Accessor interface {
 64 | 
 65 | 	// Close closes the store, rendering it unusable for I/O. It returns an
 66 | 	// error, if any.
 67 | 	Close() error
 68 | 
 69 | 	// Name returns the name of the file as presented to Open.
 70 | 	Name() string
 71 | 
 72 | 	// ReadAt reads len(b) bytes from the store starting at byte offset off.
 73 | 	// It returns the number of bytes read and the error, if any.
 74 | 	// EOF is signaled by a zero count with err set to os.EOF.
 75 | 	// ReadAt always returns a non-nil Error when n != len(b).
 76 | 	ReadAt(b []byte, off int64) (n int, err error)
 77 | 
 78 | 	// Stat returns the FileInfo structure describing the store. It returns
 79 | 	// the os.FileInfo and an error, if any.
 80 | 	Stat() (fi os.FileInfo, err error)
 81 | 
 82 | 	// Sync commits the current contents of the store to stable storage.
 83 | 	// Typically, this means flushing the file system's in-memory copy of
 84 | 	// recently written data to disk.
 85 | 	Sync() (err error)
 86 | 
 87 | 	// Truncate changes the size of the store. It does not change the I/O
 88 | 	// offset.
 89 | 	Truncate(size int64) error
 90 | 
 91 | 	// WriteAt writes len(b) bytes to the store starting at byte offset off.
 92 | 	// It returns the number of bytes written and an error, if any.
 93 | 	// WriteAt returns a non-nil Error when n != len(b).
 94 | 	WriteAt(b []byte, off int64) (n int, err error)
 95 | 
 96 | 	// Before every [structural] change of a store the BeginUpdate is to be
 97 | 	// called and paired with EndUpdate after the change makes the store's
 98 | 	// state consistent again. Invocations of BeginUpdate may nest. On
 99 | 	// invoking the last non nested EndUpdate an implicit "commit" should
100 | 	// be performed by the store/provider. The concrete mechanism is
101 | 	// unspecified. It could be for example a write-ahead log.  Stores may
102 | 	// implement BeginUpdate and EndUpdate as a (documented) no op.
103 | 	BeginUpdate() error
104 | 	EndUpdate() error
105 | }
106 | 
107 | // Mutate is a helper/wrapper for executing f in between a.BeginUpdate and
108 | // a.EndUpdate.  Any parameters and/or return values except an error should be
109 | // captured by a function literal passed as f. The returned err is either nil
110 | // or the first non nil error returned from the sequence of execution:
111 | // BeginUpdate, [f,] EndUpdate. The pair BeginUpdate/EndUpdate *is* invoked
112 | // always regardles of any possible errors produced.  Mutate doesn't handle
113 | // panic, it should be used only with a function [literal] which doesn't panic.
114 | // Otherwise the pairing of BeginUpdate/EndUpdate is not guaranteed.
115 | //
116 | // NOTE: If BeginUpdate, which is invoked before f, returns a non-nil error,
117 | // then f is not invoked at all (but EndUpdate still is).
118 | func Mutate(a Accessor, f func() error) (err error) {
119 | 	defer func() {
120 | 		if e := a.EndUpdate(); e != nil && err == nil {
121 | 			err = e
122 | 		}
123 | 	}()
124 | 
125 | 	if err = a.BeginUpdate(); err != nil {
126 | 		return
127 | 	}
128 | 
129 | 	return f()
130 | }
131 | 
132 | // LockedMutate wraps Mutate in yet another layer consisting of a
133 | // l.Lock/l.Unlock pair. All other limitations apply as in Mutate, e.g. no
134 | // panics are allowed to happen - otherwise no guarantees can be made about
135 | // Unlock matching the Lock.
136 | func LockedMutate(a Accessor, l sync.Locker, f func() error) (err error) {
137 | 	l.Lock()
138 | 	defer l.Unlock()
139 | 
140 | 	return Mutate(a, f)
141 | }
142 | 


--------------------------------------------------------------------------------
/storage/test_deps.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package storage
 8 | 
 9 | // Pull test dependencies too.
10 | // Enables easy 'go test X' after 'go get X'
11 | import (
12 | // nothing yet
13 | )
14 | 


--------------------------------------------------------------------------------
/test_deps.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2014 The fileutil Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // blame: jnml, labs.nic.cz
 6 | 
 7 | package fileutil
 8 | 
 9 | // Pull test dependencies too.
10 | // Enables easy 'go test X' after 'go get X'
11 | import (
12 | // nothing yet
13 | )
14 | 


--------------------------------------------------------------------------------