├── go.mod ├── go.sum ├── testdata └── fastwalk │ ├── fastwalk_dirent_fileno.go │ ├── fastwalk_dirent_ino.go │ ├── fastwalk_dirent_namlen_bsd.go │ ├── fastwalk_dirent_namlen_linux.go │ ├── fastwalk_portable.go │ ├── fastwalk_unix.go │ ├── fastwalk.go │ └── fastwalk_test.go ├── walker_portable.go ├── walker_option.go ├── LICENSE ├── walker_unix.go ├── walker.go ├── README.md └── walker_test.go /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/saracen/walker 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.6.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= 2 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 3 | -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk_dirent_fileno.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build freebsd || openbsd || netbsd 6 | // +build freebsd openbsd netbsd 7 | 8 | package fastwalk 9 | 10 | import "syscall" 11 | 12 | func direntInode(dirent *syscall.Dirent) uint64 { 13 | return uint64(dirent.Fileno) 14 | } 15 | -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk_dirent_ino.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build (linux || darwin) && !appengine 6 | // +build linux darwin 7 | // +build !appengine 8 | 9 | package fastwalk 10 | 11 | import "syscall" 12 | 13 | func direntInode(dirent *syscall.Dirent) uint64 { 14 | return uint64(dirent.Ino) 15 | } 16 | -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk_dirent_namlen_bsd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build darwin || freebsd || openbsd || netbsd 6 | // +build darwin freebsd openbsd netbsd 7 | 8 | package fastwalk 9 | 10 | import "syscall" 11 | 12 | func direntNamlen(dirent *syscall.Dirent) uint64 { 13 | return uint64(dirent.Namlen) 14 | } 15 | -------------------------------------------------------------------------------- /walker_portable.go: -------------------------------------------------------------------------------- 1 | // +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd 2 | 3 | package walker 4 | 5 | import "os" 6 | 7 | func (w *walker) readdir(dirname string) error { 8 | f, err := os.Open(dirname) 9 | if err != nil { 10 | return err 11 | } 12 | 13 | list, err := f.Readdir(-1) 14 | f.Close() 15 | if err != nil { 16 | return err 17 | } 18 | 19 | for _, fi := range list { 20 | if err = w.walk(dirname, fi); err != nil { 21 | return err 22 | } 23 | } 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /walker_option.go: -------------------------------------------------------------------------------- 1 | package walker 2 | 3 | // WalkerOption is an option to configure Walk() behaviour. 4 | type Option func(*walkerOptions) error 5 | 6 | type walkerOptions struct { 7 | errorCallback func(pathname string, err error) error 8 | limit int 9 | } 10 | 11 | // WithErrorCallback sets a callback to be used for error handling. Any error 12 | // returned will halt the Walk function and return the error. If the callback 13 | // returns nil Walk will continue. 14 | func WithErrorCallback(callback func(pathname string, err error) error) Option { 15 | return func(o *walkerOptions) error { 16 | o.errorCallback = callback 17 | return nil 18 | } 19 | } 20 | 21 | // WithLimit sets the maximum number of goroutines used for walking. 22 | func WithLimit(limit int) Option { 23 | return func(o *walkerOptions) error { 24 | o.limit = limit 25 | return nil 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk_dirent_namlen_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build linux && !appengine 6 | // +build linux,!appengine 7 | 8 | package fastwalk 9 | 10 | import ( 11 | "bytes" 12 | "syscall" 13 | "unsafe" 14 | ) 15 | 16 | func direntNamlen(dirent *syscall.Dirent) uint64 { 17 | const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name)) 18 | nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) 19 | const nameBufLen = uint16(len(nameBuf)) 20 | limit := dirent.Reclen - fixedHdr 21 | if limit > nameBufLen { 22 | limit = nameBufLen 23 | } 24 | nameLen := bytes.IndexByte(nameBuf[:limit], 0) 25 | if nameLen < 0 { 26 | panic("failed to find terminating 0 byte in dirent") 27 | } 28 | return uint64(nameLen) 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Arran Walker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk_portable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd) 6 | // +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd 7 | 8 | package fastwalk 9 | 10 | import ( 11 | "io/ioutil" 12 | "os" 13 | ) 14 | 15 | // readDir calls fn for each directory entry in dirName. 16 | // It does not descend into directories or follow symlinks. 17 | // If fn returns a non-nil error, readDir returns with that error 18 | // immediately. 19 | func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { 20 | fis, err := ioutil.ReadDir(dirName) 21 | if err != nil { 22 | return err 23 | } 24 | skipFiles := false 25 | for _, fi := range fis { 26 | if fi.Mode().IsRegular() && skipFiles { 27 | continue 28 | } 29 | if err := fn(dirName, fi.Name(), fi.Mode()&os.ModeType); err != nil { 30 | if err == ErrSkipFiles { 31 | skipFiles = true 32 | continue 33 | } 34 | return err 35 | } 36 | } 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /walker_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin freebsd openbsd netbsd 2 | // +build !appengine 3 | 4 | package walker 5 | 6 | import ( 7 | "os" 8 | "syscall" 9 | ) 10 | 11 | func (w *walker) readdir(dirname string) error { 12 | fd, err := open(dirname, 0, 0) 13 | if err != nil { 14 | return &os.PathError{Op: "open", Path: dirname, Err: err} 15 | } 16 | defer syscall.Close(fd) 17 | 18 | buf := make([]byte, 8<<10) 19 | names := make([]string, 0, 100) 20 | 21 | nbuf := 0 22 | bufp := 0 23 | for { 24 | if bufp >= nbuf { 25 | bufp = 0 26 | nbuf, err = readDirent(fd, buf) 27 | if err != nil { 28 | return err 29 | } 30 | if nbuf <= 0 { 31 | return nil 32 | } 33 | } 34 | 35 | consumed, count, names := syscall.ParseDirent(buf[bufp:nbuf], 100, names[0:]) 36 | bufp += consumed 37 | 38 | for _, name := range names[:count] { 39 | fi, err := os.Lstat(dirname + "/" + name) 40 | if os.IsNotExist(err) { 41 | continue 42 | } 43 | if err != nil { 44 | return err 45 | } 46 | if err = w.walk(dirname, fi); err != nil { 47 | return err 48 | } 49 | } 50 | } 51 | // never reach 52 | } 53 | 54 | // According to https://golang.org/doc/go1.14#runtime 55 | // A consequence of the implementation of preemption is that on Unix systems, including Linux and macOS 56 | // systems, programs built with Go 1.14 will receive more signals than programs built with earlier releases. 57 | // 58 | // This causes syscall.Open and syscall.ReadDirent sometimes fail with EINTR errors. 59 | // We need to retry in this case. 60 | func open(path string, mode int, perm uint32) (fd int, err error) { 61 | for { 62 | fd, err := syscall.Open(path, mode, perm) 63 | if err != syscall.EINTR { 64 | return fd, err 65 | } 66 | } 67 | } 68 | 69 | func readDirent(fd int, buf []byte) (n int, err error) { 70 | for { 71 | nbuf, err := syscall.ReadDirent(fd, buf) 72 | if err != syscall.EINTR { 73 | return nbuf, err 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /walker.go: -------------------------------------------------------------------------------- 1 | package walker 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | "sync/atomic" 9 | 10 | "golang.org/x/sync/errgroup" 11 | ) 12 | 13 | // Walk wraps WalkWithContext using the background context. 14 | func Walk(root string, walkFn func(pathname string, fi os.FileInfo) error, opts ...Option) error { 15 | return WalkWithContext(context.Background(), root, walkFn, opts...) 16 | } 17 | 18 | // WalkWithContext walks the file tree rooted at root, calling walkFn for each 19 | // file or directory in the tree, including root. 20 | // 21 | // If fastWalk returns filepath.SkipDir, the directory is skipped. 22 | // 23 | // Multiple goroutines stat the filesystem concurrently. The provided 24 | // walkFn must be safe for concurrent use. 25 | func WalkWithContext(ctx context.Context, root string, walkFn func(pathname string, fi os.FileInfo) error, opts ...Option) error { 26 | wg, ctx := errgroup.WithContext(ctx) 27 | 28 | fi, err := os.Lstat(root) 29 | if err != nil { 30 | return err 31 | } 32 | if err = walkFn(root, fi); err == filepath.SkipDir { 33 | return nil 34 | } 35 | if err != nil || !fi.IsDir() { 36 | return err 37 | } 38 | 39 | cpuLimit := runtime.GOMAXPROCS(-1) 40 | if cpuLimit < 4 { 41 | cpuLimit = 4 42 | } 43 | 44 | w := walker{ 45 | counter: 1, 46 | options: walkerOptions{ 47 | limit: cpuLimit, 48 | }, 49 | ctx: ctx, 50 | wg: wg, 51 | fn: walkFn, 52 | } 53 | 54 | for _, o := range opts { 55 | err := o(&w.options) 56 | if err != nil { 57 | return err 58 | } 59 | } 60 | 61 | w.wg.Go(func() error { 62 | return w.gowalk(root) 63 | }) 64 | 65 | return w.wg.Wait() 66 | } 67 | 68 | type walker struct { 69 | counter uint32 70 | ctx context.Context 71 | wg *errgroup.Group 72 | fn func(pathname string, fi os.FileInfo) error 73 | options walkerOptions 74 | } 75 | 76 | func (w *walker) walk(dirname string, fi os.FileInfo) error { 77 | pathname := dirname + string(filepath.Separator) + fi.Name() 78 | 79 | err := w.fn(pathname, fi) 80 | if err == filepath.SkipDir { 81 | return nil 82 | } 83 | if err != nil { 84 | return err 85 | } 86 | 87 | // don't follow symbolic links 88 | if fi.Mode()&os.ModeSymlink != 0 { 89 | return nil 90 | } 91 | 92 | if !fi.IsDir() { 93 | return nil 94 | } 95 | 96 | if err = w.ctx.Err(); err != nil { 97 | return err 98 | } 99 | 100 | current := atomic.LoadUint32(&w.counter) 101 | 102 | // if we haven't reached our goroutine limit, spawn a new one 103 | if current < uint32(w.options.limit) { 104 | if atomic.CompareAndSwapUint32(&w.counter, current, current+1) { 105 | w.wg.Go(func() error { 106 | return w.gowalk(pathname) 107 | }) 108 | return nil 109 | } 110 | } 111 | 112 | // if we've reached our limit, continue with this goroutine 113 | err = w.readdir(pathname) 114 | if err != nil && w.options.errorCallback != nil { 115 | err = w.options.errorCallback(pathname, err) 116 | } 117 | return err 118 | } 119 | 120 | func (w *walker) gowalk(pathname string) error { 121 | err := w.readdir(pathname) 122 | if err != nil && w.options.errorCallback != nil { 123 | err = w.options.errorCallback(pathname, err) 124 | } 125 | 126 | atomic.AddUint32(&w.counter, ^uint32(0)) 127 | return err 128 | } 129 | -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build (linux || darwin || freebsd || openbsd || netbsd) && !appengine 6 | // +build linux darwin freebsd openbsd netbsd 7 | // +build !appengine 8 | 9 | package fastwalk 10 | 11 | import ( 12 | "fmt" 13 | "os" 14 | "syscall" 15 | "unsafe" 16 | ) 17 | 18 | const blockSize = 8 << 10 19 | 20 | // unknownFileMode is a sentinel (and bogus) os.FileMode 21 | // value used to represent a syscall.DT_UNKNOWN Dirent.Type. 22 | const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice 23 | 24 | func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { 25 | fd, err := open(dirName, 0, 0) 26 | if err != nil { 27 | return &os.PathError{Op: "open", Path: dirName, Err: err} 28 | } 29 | defer syscall.Close(fd) 30 | 31 | // The buffer must be at least a block long. 32 | buf := make([]byte, blockSize) // stack-allocated; doesn't escape 33 | bufp := 0 // starting read position in buf 34 | nbuf := 0 // end valid data in buf 35 | skipFiles := false 36 | for { 37 | if bufp >= nbuf { 38 | bufp = 0 39 | nbuf, err = readDirent(fd, buf) 40 | if err != nil { 41 | return os.NewSyscallError("readdirent", err) 42 | } 43 | if nbuf <= 0 { 44 | return nil 45 | } 46 | } 47 | consumed, name, typ := parseDirEnt(buf[bufp:nbuf]) 48 | bufp += consumed 49 | if name == "" || name == "." || name == ".." { 50 | continue 51 | } 52 | // Fallback for filesystems (like old XFS) that don't 53 | // support Dirent.Type and have DT_UNKNOWN (0) there 54 | // instead. 55 | if typ == unknownFileMode { 56 | fi, err := os.Lstat(dirName + "/" + name) 57 | if err != nil { 58 | // It got deleted in the meantime. 59 | if os.IsNotExist(err) { 60 | continue 61 | } 62 | return err 63 | } 64 | typ = fi.Mode() & os.ModeType 65 | } 66 | if skipFiles && typ.IsRegular() { 67 | continue 68 | } 69 | if err := fn(dirName, name, typ); err != nil { 70 | if err == ErrSkipFiles { 71 | skipFiles = true 72 | continue 73 | } 74 | return err 75 | } 76 | } 77 | } 78 | 79 | func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode) { 80 | // golang.org/issue/37269 81 | dirent := &syscall.Dirent{} 82 | copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(dirent))[:], buf) 83 | if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v { 84 | panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v)) 85 | } 86 | if len(buf) < int(dirent.Reclen) { 87 | panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen)) 88 | } 89 | consumed = int(dirent.Reclen) 90 | if direntInode(dirent) == 0 { // File absent in directory. 91 | return 92 | } 93 | switch dirent.Type { 94 | case syscall.DT_REG: 95 | typ = 0 96 | case syscall.DT_DIR: 97 | typ = os.ModeDir 98 | case syscall.DT_LNK: 99 | typ = os.ModeSymlink 100 | case syscall.DT_BLK: 101 | typ = os.ModeDevice 102 | case syscall.DT_FIFO: 103 | typ = os.ModeNamedPipe 104 | case syscall.DT_SOCK: 105 | typ = os.ModeSocket 106 | case syscall.DT_UNKNOWN: 107 | typ = unknownFileMode 108 | default: 109 | // Skip weird things. 110 | // It's probably a DT_WHT (http://lwn.net/Articles/325369/) 111 | // or something. Revisit if/when this package is moved outside 112 | // of goimports. goimports only cares about regular files, 113 | // symlinks, and directories. 114 | return 115 | } 116 | 117 | nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) 118 | nameLen := direntNamlen(dirent) 119 | 120 | // Special cases for common things: 121 | if nameLen == 1 && nameBuf[0] == '.' { 122 | name = "." 123 | } else if nameLen == 2 && nameBuf[0] == '.' && nameBuf[1] == '.' { 124 | name = ".." 125 | } else { 126 | name = string(nameBuf[:nameLen]) 127 | } 128 | return 129 | } 130 | 131 | // According to https://golang.org/doc/go1.14#runtime 132 | // A consequence of the implementation of preemption is that on Unix systems, including Linux and macOS 133 | // systems, programs built with Go 1.14 will receive more signals than programs built with earlier releases. 134 | // 135 | // This causes syscall.Open and syscall.ReadDirent sometimes fail with EINTR errors. 136 | // We need to retry in this case. 137 | func open(path string, mode int, perm uint32) (fd int, err error) { 138 | for { 139 | fd, err := syscall.Open(path, mode, perm) 140 | if err != syscall.EINTR { 141 | return fd, err 142 | } 143 | } 144 | } 145 | 146 | func readDirent(fd int, buf []byte) (n int, err error) { 147 | for { 148 | nbuf, err := syscall.ReadDirent(fd, buf) 149 | if err != syscall.EINTR { 150 | return nbuf, err 151 | } 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # walker 2 | 3 | [![](https://godoc.org/github.com/saracen/walker?status.svg)](http://godoc.org/github.com/saracen/walker) 4 | 5 | `walker` is a faster, parallel version, of `filepath.Walk`. 6 | 7 | ```go 8 | // walk function called for every path found 9 | walkFn := func(pathname string, fi os.FileInfo) error { 10 | fmt.Printf("%s: %d bytes\n", pathname, fi.Size()) 11 | return nil 12 | } 13 | 14 | // error function called for every error encountered 15 | errorCallbackOption := walker.WithErrorCallback(func(pathname string, err error) error { 16 | // ignore permissione errors 17 | if os.IsPermission(err) { 18 | return nil 19 | } 20 | // halt traversal on any other error 21 | return err 22 | }) 23 | 24 | walker.Walk("/tmp", walkFn, errorCallbackOption) 25 | ``` 26 | 27 | ## Benchmarks 28 | 29 | - Standard library (`filepath.Walk`) is `FilepathWalk`. 30 | - This library is `WalkerWalk` 31 | - `FastwalkWalk` is [fastwalk](https://github.com/golang/tools/tree/master/internal/fastwalk). 32 | - `GodirwalkWalk` is [godirwalk](https://github.com/karrick/godirwalk). 33 | 34 | This library and `filepath.Walk` both perform `os.Lstat` calls and provide a full `os.FileInfo` structure to the callback. `BenchmarkFastwalkWalkLstat` and `BenchmarkGodirwalkWalkLstat` include this stat call for better comparison with `BenchmarkFilepathWalk` and `BenchmarkWalkerWalk`. 35 | 36 | This library and `fastwalk` both require the callback to be safe for concurrent use. `BenchmarkFilepathWalkAppend`, `BenchmarkWalkerWalkAppend`, `BenchmarkFastwalkWalkAppend` and `BenchmarkGodirwalkWalkAppend` append the paths found to a string slice. The callback, for the libraries that require it, use a mutex, for better comparison with the libraries that require no locking. 37 | 38 | This library will not always be the best/fastest option. In general, if you're on Windows, or performing `lstat` calls, it does a pretty decent job. If you're not, I've found `fastwalk` to perform better on machines with fewer cores. 39 | 40 | These benchmarks were performed with a warm cache. 41 | 42 | ``` 43 | goos: linux 44 | goarch: amd64 45 | pkg: github.com/saracen/walker 46 | BenchmarkFilepathWalk-16 1 1437919955 ns/op 340100304 B/op 775525 allocs/op 47 | BenchmarkFilepathWalkAppend-16 1 1226169600 ns/op 351722832 B/op 775556 allocs/op 48 | BenchmarkWalkerWalk-16 8 133364860 ns/op 92611308 B/op 734674 allocs/op 49 | BenchmarkWalkerWalkAppend-16 7 166917499 ns/op 104231474 B/op 734693 allocs/op 50 | BenchmarkFastwalkWalk-16 6 241763690 ns/op 25257176 B/op 309423 allocs/op 51 | BenchmarkFastwalkWalkAppend-16 4 285673715 ns/op 36898800 B/op 309456 allocs/op 52 | BenchmarkFastwalkWalkLstat-16 6 176641625 ns/op 73769765 B/op 592980 allocs/op 53 | BenchmarkGodirwalkWalk-16 2 714625929 ns/op 145340576 B/op 723225 allocs/op 54 | BenchmarkGodirwalkWalkAppend-16 2 597653802 ns/op 156963288 B/op 723256 allocs/op 55 | BenchmarkGodirwalkWalkLstat-16 1 1186956102 ns/op 193724464 B/op 1006727 allocs/op 56 | ``` 57 | 58 | ``` 59 | goos: windows 60 | goarch: amd64 61 | pkg: github.com/saracen/walker 62 | BenchmarkFilepathWalk-16 1 1268606000 ns/op 101248040 B/op 650718 allocs/op 63 | BenchmarkFilepathWalkAppend-16 1 1276617400 ns/op 107079288 B/op 650744 allocs/op 64 | BenchmarkWalkerWalk-16 12 98901983 ns/op 52393125 B/op 382836 allocs/op 65 | BenchmarkWalkerWalkAppend-16 12 99733117 ns/op 58220869 B/op 382853 allocs/op 66 | BenchmarkFastwalkWalk-16 10 109107980 ns/op 53032702 B/op 401320 allocs/op 67 | BenchmarkFastwalkWalkAppend-16 10 107512330 ns/op 58853827 B/op 401336 allocs/op 68 | BenchmarkFastwalkWalkLstat-16 3 379318333 ns/op 100606232 B/op 653931 allocs/op 69 | BenchmarkGodirwalkWalk-16 3 466418533 ns/op 42955197 B/op 579974 allocs/op 70 | BenchmarkGodirwalkWalkAppend-16 3 476391833 ns/op 48786530 B/op 580002 allocs/op 71 | BenchmarkGodirwalkWalkLstat-16 1 1250652800 ns/op 90536184 B/op 832562 allocs/op 72 | ``` 73 | 74 | Performing benchmarks without having the OS cache the directory information isn't straight forward, but to get a sense of the performance, we can flush the cache and roughly time how long it took to walk a directory: 75 | 76 | #### filepath.Walk 77 | ``` 78 | $ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestFilepathWalkDir -benchdir $GOPATH 79 | ok github.com/saracen/walker 3.846s 80 | ``` 81 | 82 | #### walker 83 | ``` 84 | $ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestWalkerWalkDir -benchdir $GOPATH 85 | ok github.com/saracen/walker 0.353s 86 | ``` 87 | 88 | #### fastwalk 89 | ``` 90 | $ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestFastwalkWalkDir -benchdir $GOPATH 91 | ok github.com/saracen/walker 0.306s 92 | ``` 93 | 94 | #### fastwalk (lstat) 95 | ``` 96 | $ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestFastwalkWalkLstatDir -benchdir $GOPATH 97 | ok github.com/saracen/walker 0.339s 98 | ``` 99 | 100 | #### godirwalk 101 | ``` 102 | $ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestGodirwalkWalkDir -benchdir $GOPATH 103 | ok github.com/saracen/walker 3.208s 104 | ``` 105 | -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package fastwalk provides a faster version of filepath.Walk for file system 6 | // scanning tools. 7 | package fastwalk 8 | 9 | import ( 10 | "errors" 11 | "os" 12 | "path/filepath" 13 | "runtime" 14 | "sync" 15 | ) 16 | 17 | // ErrTraverseLink is used as a return value from WalkFuncs to indicate that the 18 | // symlink named in the call may be traversed. 19 | var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory") 20 | 21 | // ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the 22 | // callback should not be called for any other files in the current directory. 23 | // Child directories will still be traversed. 24 | var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory") 25 | 26 | // Walk is a faster implementation of filepath.Walk. 27 | // 28 | // filepath.Walk's design necessarily calls os.Lstat on each file, 29 | // even if the caller needs less info. 30 | // Many tools need only the type of each file. 31 | // On some platforms, this information is provided directly by the readdir 32 | // system call, avoiding the need to stat each file individually. 33 | // fastwalk_unix.go contains a fork of the syscall routines. 34 | // 35 | // See golang.org/issue/16399 36 | // 37 | // Walk walks the file tree rooted at root, calling walkFn for 38 | // each file or directory in the tree, including root. 39 | // 40 | // If fastWalk returns filepath.SkipDir, the directory is skipped. 41 | // 42 | // Unlike filepath.Walk: 43 | // * file stat calls must be done by the user. 44 | // The only provided metadata is the file type, which does not include 45 | // any permission bits. 46 | // * multiple goroutines stat the filesystem concurrently. The provided 47 | // walkFn must be safe for concurrent use. 48 | // * fastWalk can follow symlinks if walkFn returns the TraverseLink 49 | // sentinel error. It is the walkFn's responsibility to prevent 50 | // fastWalk from going into symlink cycles. 51 | func Walk(root string, walkFn func(path string, typ os.FileMode) error) error { 52 | // TODO(bradfitz): make numWorkers configurable? We used a 53 | // minimum of 4 to give the kernel more info about multiple 54 | // things we want, in hopes its I/O scheduling can take 55 | // advantage of that. Hopefully most are in cache. Maybe 4 is 56 | // even too low of a minimum. Profile more. 57 | numWorkers := 4 58 | if n := runtime.NumCPU(); n > numWorkers { 59 | numWorkers = n 60 | } 61 | 62 | // Make sure to wait for all workers to finish, otherwise 63 | // walkFn could still be called after returning. This Wait call 64 | // runs after close(e.donec) below. 65 | var wg sync.WaitGroup 66 | defer wg.Wait() 67 | 68 | w := &walker{ 69 | fn: walkFn, 70 | enqueuec: make(chan walkItem, numWorkers), // buffered for performance 71 | workc: make(chan walkItem, numWorkers), // buffered for performance 72 | donec: make(chan struct{}), 73 | 74 | // buffered for correctness & not leaking goroutines: 75 | resc: make(chan error, numWorkers), 76 | } 77 | defer close(w.donec) 78 | 79 | for i := 0; i < numWorkers; i++ { 80 | wg.Add(1) 81 | go w.doWork(&wg) 82 | } 83 | todo := []walkItem{{dir: root}} 84 | out := 0 85 | for { 86 | workc := w.workc 87 | var workItem walkItem 88 | if len(todo) == 0 { 89 | workc = nil 90 | } else { 91 | workItem = todo[len(todo)-1] 92 | } 93 | select { 94 | case workc <- workItem: 95 | todo = todo[:len(todo)-1] 96 | out++ 97 | case it := <-w.enqueuec: 98 | todo = append(todo, it) 99 | case err := <-w.resc: 100 | out-- 101 | if err != nil { 102 | return err 103 | } 104 | if out == 0 && len(todo) == 0 { 105 | // It's safe to quit here, as long as the buffered 106 | // enqueue channel isn't also readable, which might 107 | // happen if the worker sends both another unit of 108 | // work and its result before the other select was 109 | // scheduled and both w.resc and w.enqueuec were 110 | // readable. 111 | select { 112 | case it := <-w.enqueuec: 113 | todo = append(todo, it) 114 | default: 115 | return nil 116 | } 117 | } 118 | } 119 | } 120 | } 121 | 122 | // doWork reads directories as instructed (via workc) and runs the 123 | // user's callback function. 124 | func (w *walker) doWork(wg *sync.WaitGroup) { 125 | defer wg.Done() 126 | for { 127 | select { 128 | case <-w.donec: 129 | return 130 | case it := <-w.workc: 131 | select { 132 | case <-w.donec: 133 | return 134 | case w.resc <- w.walk(it.dir, !it.callbackDone): 135 | } 136 | } 137 | } 138 | } 139 | 140 | type walker struct { 141 | fn func(path string, typ os.FileMode) error 142 | 143 | donec chan struct{} // closed on fastWalk's return 144 | workc chan walkItem // to workers 145 | enqueuec chan walkItem // from workers 146 | resc chan error // from workers 147 | } 148 | 149 | type walkItem struct { 150 | dir string 151 | callbackDone bool // callback already called; don't do it again 152 | } 153 | 154 | func (w *walker) enqueue(it walkItem) { 155 | select { 156 | case w.enqueuec <- it: 157 | case <-w.donec: 158 | } 159 | } 160 | 161 | func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error { 162 | joined := dirName + string(os.PathSeparator) + baseName 163 | if typ == os.ModeDir { 164 | w.enqueue(walkItem{dir: joined}) 165 | return nil 166 | } 167 | 168 | err := w.fn(joined, typ) 169 | if typ == os.ModeSymlink { 170 | if err == ErrTraverseLink { 171 | // Set callbackDone so we don't call it twice for both the 172 | // symlink-as-symlink and the symlink-as-directory later: 173 | w.enqueue(walkItem{dir: joined, callbackDone: true}) 174 | return nil 175 | } 176 | if err == filepath.SkipDir { 177 | // Permit SkipDir on symlinks too. 178 | return nil 179 | } 180 | } 181 | return err 182 | } 183 | 184 | func (w *walker) walk(root string, runUserCallback bool) error { 185 | if runUserCallback { 186 | err := w.fn(root, os.ModeDir) 187 | if err == filepath.SkipDir { 188 | return nil 189 | } 190 | if err != nil { 191 | return err 192 | } 193 | } 194 | 195 | return readDir(root, w.onDirEnt) 196 | } 197 | -------------------------------------------------------------------------------- /testdata/fastwalk/fastwalk_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package fastwalk_test 6 | 7 | import ( 8 | "bytes" 9 | "flag" 10 | "fmt" 11 | "io/ioutil" 12 | "os" 13 | "path/filepath" 14 | "reflect" 15 | "runtime" 16 | "sort" 17 | "strings" 18 | "sync" 19 | "testing" 20 | 21 | "github.com/saracen/walker/testdata/fastwalk" 22 | ) 23 | 24 | func formatFileModes(m map[string]os.FileMode) string { 25 | var keys []string 26 | for k := range m { 27 | keys = append(keys, k) 28 | } 29 | sort.Strings(keys) 30 | var buf bytes.Buffer 31 | for _, k := range keys { 32 | fmt.Fprintf(&buf, "%-20s: %v\n", k, m[k]) 33 | } 34 | return buf.String() 35 | } 36 | 37 | func testFastWalk(t *testing.T, files map[string]string, callback func(path string, typ os.FileMode) error, want map[string]os.FileMode) { 38 | tempdir, err := ioutil.TempDir("", "test-fast-walk") 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | defer os.RemoveAll(tempdir) 43 | 44 | symlinks := map[string]string{} 45 | for path, contents := range files { 46 | file := filepath.Join(tempdir, "/src", path) 47 | if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { 48 | t.Fatal(err) 49 | } 50 | var err error 51 | if strings.HasPrefix(contents, "LINK:") { 52 | symlinks[file] = filepath.FromSlash(strings.TrimPrefix(contents, "LINK:")) 53 | } else { 54 | err = ioutil.WriteFile(file, []byte(contents), 0644) 55 | } 56 | if err != nil { 57 | t.Fatal(err) 58 | } 59 | } 60 | 61 | // Create symlinks after all other files. Otherwise, directory symlinks on 62 | // Windows are unusable (see https://golang.org/issue/39183). 63 | for file, dst := range symlinks { 64 | err = os.Symlink(dst, file) 65 | if err != nil { 66 | if writeErr := ioutil.WriteFile(file, []byte(dst), 0644); writeErr == nil { 67 | // Couldn't create symlink, but could write the file. 68 | // Probably this filesystem doesn't support symlinks. 69 | // (Perhaps we are on an older Windows and not running as administrator.) 70 | t.Skipf("skipping because symlinks appear to be unsupported: %v", err) 71 | } 72 | } 73 | } 74 | 75 | got := map[string]os.FileMode{} 76 | var mu sync.Mutex 77 | err = fastwalk.Walk(tempdir, func(path string, typ os.FileMode) error { 78 | mu.Lock() 79 | defer mu.Unlock() 80 | if !strings.HasPrefix(path, tempdir) { 81 | t.Errorf("bogus prefix on %q, expect %q", path, tempdir) 82 | } 83 | key := filepath.ToSlash(strings.TrimPrefix(path, tempdir)) 84 | if old, dup := got[key]; dup { 85 | t.Errorf("callback called twice for key %q: %v -> %v", key, old, typ) 86 | } 87 | got[key] = typ 88 | return callback(path, typ) 89 | }) 90 | 91 | if err != nil { 92 | t.Fatalf("callback returned: %v", err) 93 | } 94 | if !reflect.DeepEqual(got, want) { 95 | t.Errorf("walk mismatch.\n got:\n%v\nwant:\n%v", formatFileModes(got), formatFileModes(want)) 96 | } 97 | } 98 | 99 | func TestFastWalk_Basic(t *testing.T) { 100 | testFastWalk(t, map[string]string{ 101 | "foo/foo.go": "one", 102 | "bar/bar.go": "two", 103 | "skip/skip.go": "skip", 104 | }, 105 | func(path string, typ os.FileMode) error { 106 | return nil 107 | }, 108 | map[string]os.FileMode{ 109 | "": os.ModeDir, 110 | "/src": os.ModeDir, 111 | "/src/bar": os.ModeDir, 112 | "/src/bar/bar.go": 0, 113 | "/src/foo": os.ModeDir, 114 | "/src/foo/foo.go": 0, 115 | "/src/skip": os.ModeDir, 116 | "/src/skip/skip.go": 0, 117 | }) 118 | } 119 | 120 | func TestFastWalk_LongFileName(t *testing.T) { 121 | longFileName := strings.Repeat("x", 255) 122 | 123 | testFastWalk(t, map[string]string{ 124 | longFileName: "one", 125 | }, 126 | func(path string, typ os.FileMode) error { 127 | return nil 128 | }, 129 | map[string]os.FileMode{ 130 | "": os.ModeDir, 131 | "/src": os.ModeDir, 132 | "/src/" + longFileName: 0, 133 | }, 134 | ) 135 | } 136 | 137 | func TestFastWalk_Symlink(t *testing.T) { 138 | testFastWalk(t, map[string]string{ 139 | "foo/foo.go": "one", 140 | "bar/bar.go": "LINK:../foo/foo.go", 141 | "symdir": "LINK:foo", 142 | "broken/broken.go": "LINK:../nonexistent", 143 | }, 144 | func(path string, typ os.FileMode) error { 145 | return nil 146 | }, 147 | map[string]os.FileMode{ 148 | "": os.ModeDir, 149 | "/src": os.ModeDir, 150 | "/src/bar": os.ModeDir, 151 | "/src/bar/bar.go": os.ModeSymlink, 152 | "/src/foo": os.ModeDir, 153 | "/src/foo/foo.go": 0, 154 | "/src/symdir": os.ModeSymlink, 155 | "/src/broken": os.ModeDir, 156 | "/src/broken/broken.go": os.ModeSymlink, 157 | }) 158 | } 159 | 160 | func TestFastWalk_SkipDir(t *testing.T) { 161 | testFastWalk(t, map[string]string{ 162 | "foo/foo.go": "one", 163 | "bar/bar.go": "two", 164 | "skip/skip.go": "skip", 165 | }, 166 | func(path string, typ os.FileMode) error { 167 | if typ == os.ModeDir && strings.HasSuffix(path, "skip") { 168 | return filepath.SkipDir 169 | } 170 | return nil 171 | }, 172 | map[string]os.FileMode{ 173 | "": os.ModeDir, 174 | "/src": os.ModeDir, 175 | "/src/bar": os.ModeDir, 176 | "/src/bar/bar.go": 0, 177 | "/src/foo": os.ModeDir, 178 | "/src/foo/foo.go": 0, 179 | "/src/skip": os.ModeDir, 180 | }) 181 | } 182 | 183 | func TestFastWalk_SkipFiles(t *testing.T) { 184 | // Directory iteration order is undefined, so there's no way to know 185 | // which file to expect until the walk happens. Rather than mess 186 | // with the test infrastructure, just mutate want. 187 | var mu sync.Mutex 188 | want := map[string]os.FileMode{ 189 | "": os.ModeDir, 190 | "/src": os.ModeDir, 191 | "/src/zzz": os.ModeDir, 192 | "/src/zzz/c.go": 0, 193 | } 194 | 195 | testFastWalk(t, map[string]string{ 196 | "a_skipfiles.go": "a", 197 | "b_skipfiles.go": "b", 198 | "zzz/c.go": "c", 199 | }, 200 | func(path string, typ os.FileMode) error { 201 | if strings.HasSuffix(path, "_skipfiles.go") { 202 | mu.Lock() 203 | defer mu.Unlock() 204 | want["/src/"+filepath.Base(path)] = 0 205 | return fastwalk.ErrSkipFiles 206 | } 207 | return nil 208 | }, 209 | want) 210 | if len(want) != 5 { 211 | t.Errorf("saw too many files: wanted 5, got %v (%v)", len(want), want) 212 | } 213 | } 214 | 215 | func TestFastWalk_TraverseSymlink(t *testing.T) { 216 | testFastWalk(t, map[string]string{ 217 | "foo/foo.go": "one", 218 | "bar/bar.go": "two", 219 | "skip/skip.go": "skip", 220 | "symdir": "LINK:foo", 221 | }, 222 | func(path string, typ os.FileMode) error { 223 | if typ == os.ModeSymlink { 224 | return fastwalk.ErrTraverseLink 225 | } 226 | return nil 227 | }, 228 | map[string]os.FileMode{ 229 | "": os.ModeDir, 230 | "/src": os.ModeDir, 231 | "/src/bar": os.ModeDir, 232 | "/src/bar/bar.go": 0, 233 | "/src/foo": os.ModeDir, 234 | "/src/foo/foo.go": 0, 235 | "/src/skip": os.ModeDir, 236 | "/src/skip/skip.go": 0, 237 | "/src/symdir": os.ModeSymlink, 238 | "/src/symdir/foo.go": 0, 239 | }) 240 | } 241 | 242 | var benchDir = flag.String("benchdir", runtime.GOROOT(), "The directory to scan for BenchmarkFastWalk") 243 | 244 | func BenchmarkFastWalk(b *testing.B) { 245 | b.ReportAllocs() 246 | for i := 0; i < b.N; i++ { 247 | err := fastwalk.Walk(*benchDir, func(path string, typ os.FileMode) error { return nil }) 248 | if err != nil { 249 | b.Fatal(err) 250 | } 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /walker_test.go: -------------------------------------------------------------------------------- 1 | package walker_test 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "strings" 11 | "sync" 12 | "testing" 13 | "time" 14 | 15 | // "github.com/karrick/godirwalk" 16 | "github.com/saracen/walker" 17 | "github.com/saracen/walker/testdata/fastwalk" 18 | ) 19 | 20 | func testWalk(t *testing.T, files map[string]os.FileMode) { 21 | dir, err := ioutil.TempDir("", "walker-test") 22 | if err != nil { 23 | t.Error(err) 24 | return 25 | } 26 | defer os.RemoveAll(dir) 27 | 28 | for path, mode := range files { 29 | path = filepath.Join(dir, path) 30 | err := os.MkdirAll(filepath.Dir(path), 0777) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | switch { 36 | case mode&os.ModeSymlink != 0 && mode&os.ModeDir != 0: 37 | err = os.Symlink(filepath.Dir(path), path) 38 | 39 | case mode&os.ModeSymlink != 0: 40 | err = os.Symlink("foo/foo.go", path) 41 | 42 | case mode&os.ModeDir != 0: 43 | err = os.Mkdir(path, mode) 44 | 45 | default: 46 | err = ioutil.WriteFile(path, []byte(path), mode) 47 | } 48 | if err != nil { 49 | t.Fatal(err) 50 | } 51 | } 52 | 53 | filepathResults := make(map[string]os.FileInfo) 54 | err = filepath.Walk(dir, func(pathname string, fi os.FileInfo, err error) error { 55 | if strings.Contains(pathname, "skip") { 56 | return filepath.SkipDir 57 | } 58 | 59 | if filepath.Base(pathname) == "perm-error" && runtime.GOOS != "windows" { 60 | if err == nil { 61 | t.Errorf("expected permission error for path %v", pathname) 62 | } 63 | } else { 64 | if err != nil { 65 | t.Errorf("unexpected error for path %v", pathname) 66 | } 67 | } 68 | 69 | filepathResults[pathname] = fi 70 | return nil 71 | }) 72 | if err != nil { 73 | t.Fatal(err) 74 | } 75 | 76 | var l sync.Mutex 77 | walkerResults := make(map[string]os.FileInfo) 78 | err = walker.Walk(dir, func(pathname string, fi os.FileInfo) error { 79 | if strings.Contains(pathname, "skip") { 80 | return filepath.SkipDir 81 | } 82 | 83 | l.Lock() 84 | walkerResults[pathname] = fi 85 | l.Unlock() 86 | 87 | return nil 88 | }, walker.WithErrorCallback(func(pathname string, err error) error { 89 | if filepath.Base(pathname) == "perm-error" { 90 | if err == nil { 91 | t.Errorf("expected permission error for path %v", pathname) 92 | } 93 | } else { 94 | if err != nil { 95 | t.Errorf("unexpected error for path %v", pathname) 96 | } 97 | } 98 | return nil 99 | })) 100 | 101 | if err != nil { 102 | t.Fatal(err) 103 | } 104 | 105 | for path, info := range filepathResults { 106 | info2, ok := walkerResults[path] 107 | if !ok { 108 | t.Fatalf("walk mismatch, path %q doesn't exist", path) 109 | } 110 | 111 | if info.IsDir() != info2.IsDir() || 112 | info.ModTime() != info2.ModTime() || 113 | info.Mode() != info2.Mode() || 114 | info.Name() != info2.Name() || 115 | info.Size() != info2.Size() { 116 | t.Fatalf("walk mismatch, got %v, wanted %v", info2, info) 117 | } 118 | } 119 | } 120 | 121 | func TestWalker(t *testing.T) { 122 | testWalk(t, map[string]os.FileMode{ 123 | "foo/foo.go": 0644, 124 | "bar/bar.go": 0777, 125 | "bar/foo/bar/foo/bar": 0600, 126 | "skip/file": 0700, 127 | "bar/symlink": os.ModeDir | os.ModeSymlink | 0777, 128 | "bar/symlink.go": os.ModeSymlink | 0777, 129 | "perm-error": os.ModeDir | 0000, 130 | }) 131 | } 132 | 133 | func TestWalkerWithContext(t *testing.T) { 134 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Nanosecond) 135 | defer cancel() 136 | err := walker.WalkWithContext(ctx, runtime.GOROOT(), func(pathname string, fi os.FileInfo) error { 137 | return nil 138 | }) 139 | if err == nil { 140 | t.Fatalf("expecting timeout error, got nil") 141 | } 142 | } 143 | 144 | var benchDir = flag.String("benchdir", runtime.GOROOT(), "The directory to scan for BenchmarkFilepathWalk and BenchmarkWalkerWalk") 145 | 146 | type tester interface { 147 | Fatal(args ...interface{}) 148 | } 149 | 150 | func filepathWalk(t tester) { 151 | err := filepath.Walk(*benchDir, func(pathname string, fi os.FileInfo, err error) error { return nil }) 152 | if err != nil { 153 | t.Fatal(err) 154 | } 155 | } 156 | 157 | func filepathWalkAppend(t tester) (paths []string) { 158 | err := filepath.Walk(*benchDir, func(pathname string, fi os.FileInfo, err error) error { 159 | paths = append(paths, pathname) 160 | return nil 161 | }) 162 | if err != nil { 163 | t.Fatal(err) 164 | } 165 | return 166 | } 167 | 168 | func TestFilepathWalkDir(t *testing.T) { filepathWalk(t) } 169 | 170 | func BenchmarkFilepathWalk(b *testing.B) { 171 | b.ReportAllocs() 172 | for i := 0; i < b.N; i++ { 173 | filepathWalk(b) 174 | } 175 | } 176 | 177 | func BenchmarkFilepathWalkAppend(b *testing.B) { 178 | b.ReportAllocs() 179 | for i := 0; i < b.N; i++ { 180 | _ = filepathWalkAppend(b) 181 | } 182 | } 183 | 184 | func walkerWalk(t tester) { 185 | err := walker.Walk(*benchDir, func(pathname string, fi os.FileInfo) error { return nil }) 186 | if err != nil { 187 | t.Fatal(err) 188 | } 189 | } 190 | 191 | func walkerWalkAppend(t tester) (paths []string) { 192 | var l sync.Mutex 193 | err := walker.Walk(*benchDir, func(pathname string, fi os.FileInfo) error { 194 | l.Lock() 195 | paths = append(paths, pathname) 196 | l.Unlock() 197 | return nil 198 | }) 199 | if err != nil { 200 | t.Fatal(err) 201 | } 202 | return 203 | } 204 | 205 | func TestWalkerWalkDir(t *testing.T) { walkerWalk(t) } 206 | 207 | func BenchmarkWalkerWalk(b *testing.B) { 208 | b.ReportAllocs() 209 | for i := 0; i < b.N; i++ { 210 | walkerWalk(b) 211 | } 212 | } 213 | 214 | func BenchmarkWalkerWalkAppend(b *testing.B) { 215 | b.ReportAllocs() 216 | for i := 0; i < b.N; i++ { 217 | _ = walkerWalkAppend(b) 218 | } 219 | } 220 | 221 | func fastwalkWalk(t tester) { 222 | err := fastwalk.Walk(*benchDir, func(pathname string, mode os.FileMode) error { return nil }) 223 | if err != nil { 224 | t.Fatal(err) 225 | } 226 | } 227 | 228 | func fastwalkWalkLstat(t tester) { 229 | err := fastwalk.Walk(*benchDir, func(pathname string, mode os.FileMode) error { 230 | _, err := os.Lstat(pathname) 231 | return err 232 | }) 233 | if err != nil { 234 | t.Fatal(err) 235 | } 236 | } 237 | 238 | func fastwalkWalkAppend(t tester) (paths []string) { 239 | var l sync.Mutex 240 | err := fastwalk.Walk(*benchDir, func(pathname string, mode os.FileMode) error { 241 | l.Lock() 242 | paths = append(paths, pathname) 243 | l.Unlock() 244 | return nil 245 | }) 246 | if err != nil { 247 | t.Fatal(err) 248 | } 249 | return 250 | } 251 | 252 | func TestFastwalkWalkDir(t *testing.T) { fastwalkWalk(t) } 253 | 254 | func TestFastwalkWalkLstatDir(t *testing.T) { fastwalkWalkLstat(t) } 255 | 256 | func BenchmarkFastwalkWalk(b *testing.B) { 257 | b.ReportAllocs() 258 | for i := 0; i < b.N; i++ { 259 | fastwalkWalk(b) 260 | } 261 | } 262 | 263 | func BenchmarkFastwalkWalkAppend(b *testing.B) { 264 | b.ReportAllocs() 265 | for i := 0; i < b.N; i++ { 266 | _ = fastwalkWalkAppend(b) 267 | } 268 | } 269 | 270 | func BenchmarkFastwalkWalkLstat(b *testing.B) { 271 | b.ReportAllocs() 272 | for i := 0; i < b.N; i++ { 273 | fastwalkWalkLstat(b) 274 | } 275 | } 276 | 277 | /*func godirwalkWalk(t tester) { 278 | err := godirwalk.Walk(*benchDir, &godirwalk.Options{ 279 | Callback: func(osPathname string, dirent *godirwalk.Dirent) error { 280 | return nil 281 | }, 282 | Unsorted: true, 283 | }) 284 | if err != nil { 285 | t.Fatal(err) 286 | } 287 | } 288 | 289 | func godirwalkWalkLstat(t tester) (paths []string) { 290 | err := godirwalk.Walk(*benchDir, &godirwalk.Options{ 291 | Callback: func(osPathname string, dirent *godirwalk.Dirent) error { 292 | _, err := os.Lstat(osPathname) 293 | return err 294 | }, 295 | Unsorted: true, 296 | }) 297 | if err != nil { 298 | t.Fatal(err) 299 | } 300 | return 301 | } 302 | 303 | func godirwalkWalkAppend(t tester) (paths []string) { 304 | err := godirwalk.Walk(*benchDir, &godirwalk.Options{ 305 | Callback: func(osPathname string, dirent *godirwalk.Dirent) error { 306 | paths = append(paths, osPathname) 307 | return nil 308 | }, 309 | Unsorted: true, 310 | }) 311 | if err != nil { 312 | t.Fatal(err) 313 | } 314 | return 315 | } 316 | 317 | func TestGodirwalkWalkDir(t *testing.T) { godirwalkWalk(t) } 318 | 319 | func BenchmarkGodirwalkWalk(b *testing.B) { 320 | b.ReportAllocs() 321 | for i := 0; i < b.N; i++ { 322 | godirwalkWalk(b) 323 | } 324 | } 325 | 326 | func BenchmarkGodirwalkWalkAppend(b *testing.B) { 327 | b.ReportAllocs() 328 | for i := 0; i < b.N; i++ { 329 | _ = godirwalkWalkAppend(b) 330 | } 331 | } 332 | 333 | func BenchmarkGodirwalkWalkLstat(b *testing.B) { 334 | b.ReportAllocs() 335 | for i := 0; i < b.N; i++ { 336 | godirwalkWalkLstat(b) 337 | } 338 | }*/ 339 | --------------------------------------------------------------------------------