├── .gitignore ├── LICENSE ├── README.md ├── azure-pipelines.yml ├── bench.sh ├── benchmark_test.go ├── debug_development.go ├── debug_release.go ├── dirent.go ├── dirent_test.go ├── doc.go ├── ensure_test.go ├── examples ├── find-fast │ ├── go.mod │ ├── go.sum │ └── main.go ├── remove-empty-directories │ └── main.go ├── scanner │ └── main.go ├── sizes │ └── main.go ├── walk-fast │ └── main.go └── walk-stdlib │ └── main.go ├── go.mod ├── go.sum ├── inoWithFileno.go ├── inoWithIno.go ├── modeType.go ├── modeTypeWithType.go ├── modeTypeWithoutType.go ├── nameWithNamlen.go ├── nameWithoutNamlen.go ├── readdir.go ├── readdir_test.go ├── readdir_unix.go ├── readdir_windows.go ├── reclenFromNamlen.go ├── reclenFromReclen.go ├── scaffoling_test.go ├── scandir_test.go ├── scandir_unix.go ├── scandir_windows.go ├── scanner.go ├── walk.go └── walk_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | examples/remove-empty-directories/remove-empty-directories 17 | examples/sizes/sizes 18 | examples/walk-fast/walk-fast 19 | examples/walk-stdlib/walk-stdlib 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2017, Karrick McDermott 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # godirwalk 2 | 3 | `godirwalk` is a library for traversing a directory tree on a file 4 | system. 5 | 6 | [![GoDoc](https://godoc.org/github.com/karrick/godirwalk?status.svg)](https://godoc.org/github.com/karrick/godirwalk) [![Build Status](https://dev.azure.com/microsoft0235/microsoft/_apis/build/status/karrick.godirwalk?branchName=master)](https://dev.azure.com/microsoft0235/microsoft/_build/latest?definitionId=1&branchName=master) 7 | 8 | In short, why did I create this library? 9 | 10 | 1. It's faster than `filepath.Walk`. 11 | 1. It's more correct on Windows than `filepath.Walk`. 12 | 1. It's more easy to use than `filepath.Walk`. 13 | 1. It's more flexible than `filepath.Walk`. 14 | 15 | Depending on your specific circumstances, [you might no longer need a 16 | library for file walking in 17 | Go](https://engineering.kablamo.com.au/posts/2021/quick-comparison-between-go-file-walk-implementations). 18 | 19 | ## Usage Example 20 | 21 | Additional examples are provided in the `examples/` subdirectory. 22 | 23 | This library will normalize the provided top level directory name 24 | based on the os-specific path separator by calling `filepath.Clean` on 25 | its first argument. However it always provides the pathname created by 26 | using the correct os-specific path separator when invoking the 27 | provided callback function. 28 | 29 | ```Go 30 | dirname := "some/directory/root" 31 | err := godirwalk.Walk(dirname, &godirwalk.Options{ 32 | Callback: func(osPathname string, de *godirwalk.Dirent) error { 33 | // Following string operation is not most performant way 34 | // of doing this, but common enough to warrant a simple 35 | // example here: 36 | if strings.Contains(osPathname, ".git") { 37 | return godirwalk.SkipThis 38 | } 39 | fmt.Printf("%s %s\n", de.ModeType(), osPathname) 40 | return nil 41 | }, 42 | Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc) 43 | }) 44 | ``` 45 | 46 | This library not only provides functions for traversing a file system 47 | directory tree, but also for obtaining a list of immediate descendants 48 | of a particular directory, typically much more quickly than using 49 | `os.ReadDir` or `os.ReadDirnames`. 50 | 51 | ## Description 52 | 53 | Here's why I use `godirwalk` in preference to `filepath.Walk`, 54 | `os.ReadDir`, and `os.ReadDirnames`. 55 | 56 | ### It's faster than `filepath.Walk` 57 | 58 | When compared against `filepath.Walk` in benchmarks, it has been 59 | observed to run between five and ten times the speed on darwin, at 60 | speeds comparable to the that of the unix `find` utility; and about 61 | twice the speed on linux; and about four times the speed on Windows. 62 | 63 | How does it obtain this performance boost? It does less work to give 64 | you nearly the same output. This library calls the same `syscall` 65 | functions to do the work, but it makes fewer calls, does not throw 66 | away information that it might need, and creates less memory churn 67 | along the way by reusing the same scratch buffer for reading from a 68 | directory rather than reallocating a new buffer every time it reads 69 | file system entry data from the operating system. 70 | 71 | While traversing a file system directory tree, `filepath.Walk` obtains 72 | the list of immediate descendants of a directory, and throws away the 73 | node type information for the file system entry that is provided by 74 | the operating system that comes with the node's name. Then, 75 | immediately prior to invoking the callback function, `filepath.Walk` 76 | invokes `os.Stat` for each node, and passes the returned `os.FileInfo` 77 | information to the callback. 78 | 79 | While the `os.FileInfo` information provided by `os.Stat` is extremely 80 | helpful--and even includes the `os.FileMode` data--providing it 81 | requires an additional system call for each node. 82 | 83 | Because most callbacks only care about what the node type is, this 84 | library does not throw the type information away, but rather provides 85 | that information to the callback function in the form of a 86 | `os.FileMode` value. Note that the provided `os.FileMode` value that 87 | this library provides only has the node type information, and does not 88 | have the permission bits, sticky bits, or other information from the 89 | file's mode. If the callback does care about a particular node's 90 | entire `os.FileInfo` data structure, the callback can easiy invoke 91 | `os.Stat` when needed, and only when needed. 92 | 93 | #### Benchmarks 94 | 95 | ##### macOS 96 | 97 | ```Bash 98 | $ go test -bench=. -benchmem 99 | goos: darwin 100 | goarch: amd64 101 | pkg: github.com/karrick/godirwalk 102 | BenchmarkReadDirnamesStandardLibrary-12 50000 26250 ns/op 10360 B/op 16 allocs/op 103 | BenchmarkReadDirnamesThisLibrary-12 50000 24372 ns/op 5064 B/op 20 allocs/op 104 | BenchmarkFilepathWalk-12 1 1099524875 ns/op 228415912 B/op 416952 allocs/op 105 | BenchmarkGodirwalk-12 2 526754589 ns/op 103110464 B/op 451442 allocs/op 106 | BenchmarkGodirwalkUnsorted-12 3 509219296 ns/op 100751400 B/op 378800 allocs/op 107 | BenchmarkFlameGraphFilepathWalk-12 1 7478618820 ns/op 2284138176 B/op 4169453 allocs/op 108 | BenchmarkFlameGraphGodirwalk-12 1 4977264058 ns/op 1031105328 B/op 4514423 allocs/op 109 | PASS 110 | ok github.com/karrick/godirwalk 21.219s 111 | ``` 112 | 113 | ##### Linux 114 | 115 | ```Bash 116 | $ go test -bench=. -benchmem 117 | goos: linux 118 | goarch: amd64 119 | pkg: github.com/karrick/godirwalk 120 | BenchmarkReadDirnamesStandardLibrary-12 100000 15458 ns/op 10360 B/op 16 allocs/op 121 | BenchmarkReadDirnamesThisLibrary-12 100000 14646 ns/op 5064 B/op 20 allocs/op 122 | BenchmarkFilepathWalk-12 2 631034745 ns/op 228210216 B/op 416939 allocs/op 123 | BenchmarkGodirwalk-12 3 358714883 ns/op 102988664 B/op 451437 allocs/op 124 | BenchmarkGodirwalkUnsorted-12 3 355363915 ns/op 100629234 B/op 378796 allocs/op 125 | BenchmarkFlameGraphFilepathWalk-12 1 6086913991 ns/op 2282104720 B/op 4169417 allocs/op 126 | BenchmarkFlameGraphGodirwalk-12 1 3456398824 ns/op 1029886400 B/op 4514373 allocs/op 127 | PASS 128 | ok github.com/karrick/godirwalk 19.179s 129 | ``` 130 | 131 | ### It's more correct on Windows than `filepath.Walk` 132 | 133 | I did not previously care about this either, but humor me. We all love 134 | how we can write once and run everywhere. It is essential for the 135 | language's adoption, growth, and success, that the software we create 136 | can run unmodified on all architectures and operating systems 137 | supported by Go. 138 | 139 | When the traversed file system has a logical loop caused by symbolic 140 | links to directories, on unix `filepath.Walk` ignores symbolic links 141 | and traverses the entire directory tree without error. On Windows 142 | however, `filepath.Walk` will continue following directory symbolic 143 | links, even though it is not supposed to, eventually causing 144 | `filepath.Walk` to terminate early and return an error when the 145 | pathname gets too long from concatenating endless loops of symbolic 146 | links onto the pathname. This error comes from Windows, passes through 147 | `filepath.Walk`, and to the upstream client running `filepath.Walk`. 148 | 149 | The takeaway is that behavior is different based on which platform 150 | `filepath.Walk` is running. While this is clearly not intentional, 151 | until it is fixed in the standard library, it presents a compatibility 152 | problem. 153 | 154 | This library fixes the above problem such that it will never follow 155 | logical file sytem loops on either unix or Windows. Furthermore, it 156 | will only follow symbolic links when `FollowSymbolicLinks` is set to 157 | true. Behavior on Windows and other operating systems is identical. 158 | 159 | ### It's more easy to use than `filepath.Walk` 160 | 161 | While this library strives to mimic the behavior of the incredibly 162 | well-written `filepath.Walk` standard library, there are places where 163 | it deviates a bit in order to provide a more easy or intuitive caller 164 | interface. 165 | 166 | #### Callback interface does not send you an error to check 167 | 168 | Since this library does not invoke `os.Stat` on every file system node 169 | it encounters, there is no possible error event for the callback 170 | function to filter on. The third argument in the `filepath.WalkFunc` 171 | function signature to pass the error from `os.Stat` to the callback 172 | function is no longer necessary, and thus eliminated from signature of 173 | the callback function from this library. 174 | 175 | Furthermore, this slight interface difference between 176 | `filepath.WalkFunc` and this library's `WalkFunc` eliminates the 177 | boilerplate code that callback handlers must write when they use 178 | `filepath.Walk`. Rather than every callback function needing to check 179 | the error value passed into it and branch accordingly, users of this 180 | library do not even have an error value to check immediately upon 181 | entry into the callback function. This is an improvement both in 182 | runtime performance and code clarity. 183 | 184 | #### Callback function is invoked with OS specific file system path separator 185 | 186 | On every OS platform `filepath.Walk` invokes the callback function 187 | with a solidus (`/`) delimited pathname. By contrast this library 188 | invokes the callback with the os-specific pathname separator, 189 | obviating a call to `filepath.Clean` in the callback function for each 190 | node prior to actually using the provided pathname. 191 | 192 | In other words, even on Windows, `filepath.Walk` will invoke the 193 | callback with `some/path/to/foo.txt`, requiring well written clients 194 | to perform pathname normalization for every file prior to working with 195 | the specified file. This is a hidden boilerplate requirement to create 196 | truly os agnostic callback functions. In truth, many clients developed 197 | on unix and not tested on Windows neglect this subtlety, and will 198 | result in software bugs when someone tries to run that software on 199 | Windows. 200 | 201 | This library invokes the callback function with `some\path\to\foo.txt` 202 | for the same file when running on Windows, eliminating the need to 203 | normalize the pathname by the client, and lessen the likelyhood that a 204 | client will work on unix but not on Windows. 205 | 206 | This enhancement eliminates necessity for some more boilerplate code 207 | in callback functions while improving the runtime performance of this 208 | library. 209 | 210 | #### `godirwalk.SkipThis` is more intuitive to use than `filepath.SkipDir` 211 | 212 | One arguably confusing aspect of the `filepath.WalkFunc` interface 213 | that this library must emulate is how a caller tells the `Walk` 214 | function to skip file system entries. With both `filepath.Walk` and 215 | this library's `Walk`, when a callback function wants to skip a 216 | directory and not descend into its children, it returns 217 | `filepath.SkipDir`. If the callback function returns 218 | `filepath.SkipDir` for a non-directory, `filepath.Walk` and this 219 | library will stop processing any more entries in the current 220 | directory. This is not necessarily what most developers want or 221 | expect. If you want to simply skip a particular non-directory entry 222 | but continue processing entries in the directory, the callback 223 | function must return nil. 224 | 225 | The implications of this interface design is when you want to walk a 226 | file system hierarchy and skip an entry, you have to return a 227 | different value based on what type of file system entry that node 228 | is. To skip an entry, if the entry is a directory, you must return 229 | `filepath.SkipDir`, and if entry is not a directory, you must return 230 | `nil`. This is an unfortunate hurdle I have observed many developers 231 | struggling with, simply because it is not an intuitive interface. 232 | 233 | Here is an example callback function that adheres to 234 | `filepath.WalkFunc` interface to have it skip any file system entry 235 | whose full pathname includes a particular substring, `optSkip`. Note 236 | that this library still supports identical behavior of `filepath.Walk` 237 | when the callback function returns `filepath.SkipDir`. 238 | 239 | ```Go 240 | func callback1(osPathname string, de *godirwalk.Dirent) error { 241 | if optSkip != "" && strings.Contains(osPathname, optSkip) { 242 | if b, err := de.IsDirOrSymlinkToDir(); b == true && err == nil { 243 | return filepath.SkipDir 244 | } 245 | return nil 246 | } 247 | // Process file like normal... 248 | return nil 249 | } 250 | ``` 251 | 252 | This library attempts to eliminate some of that logic boilerplate 253 | required in callback functions by providing a new token error value, 254 | `SkipThis`, which a callback function may return to skip the current 255 | file system entry regardless of what type of entry it is. If the 256 | current entry is a directory, its children will not be enumerated, 257 | exactly as if the callback had returned `filepath.SkipDir`. If the 258 | current entry is a non-directory, the next file system entry in the 259 | current directory will be enumerated, exactly as if the callback 260 | returned `nil`. The following example callback function has identical 261 | behavior as the previous, but has less boilerplate, and admittedly 262 | logic that I find more simple to follow. 263 | 264 | ```Go 265 | func callback2(osPathname string, de *godirwalk.Dirent) error { 266 | if optSkip != "" && strings.Contains(osPathname, optSkip) { 267 | return godirwalk.SkipThis 268 | } 269 | // Process file like normal... 270 | return nil 271 | } 272 | ``` 273 | 274 | ### It's more flexible than `filepath.Walk` 275 | 276 | #### Configurable Handling of Symbolic Links 277 | 278 | The default behavior of this library is to ignore symbolic links to 279 | directories when walking a directory tree, just like `filepath.Walk` 280 | does. However, it does invoke the callback function with each node it 281 | finds, including symbolic links. If a particular use case exists to 282 | follow symbolic links when traversing a directory tree, this library 283 | can be invoked in manner to do so, by setting the 284 | `FollowSymbolicLinks` config parameter to `true`. 285 | 286 | #### Configurable Sorting of Directory Children 287 | 288 | The default behavior of this library is to always sort the immediate 289 | descendants of a directory prior to visiting each node, just like 290 | `filepath.Walk` does. This is usually the desired behavior. However, 291 | this does come at slight performance and memory penalties required to 292 | sort the names when a directory node has many entries. Additionally if 293 | caller specifies `Unsorted` enumeration in the configuration 294 | parameter, reading directories is lazily performed as the caller 295 | consumes entries. If a particular use case exists that does not 296 | require sorting the directory's immediate descendants prior to 297 | visiting its nodes, this library will skip the sorting step when the 298 | `Unsorted` parameter is set to `true`. 299 | 300 | Here's an interesting read of the potential hazzards of traversing a 301 | file system hierarchy in a non-deterministic order. If you know the 302 | problem you are solving is not affected by the order files are 303 | visited, then I encourage you to use `Unsorted`. Otherwise skip 304 | setting this option. 305 | 306 | [Researchers find bug in Python script may have affected hundreds of studies](https://arstechnica.com/information-technology/2019/10/chemists-discover-cross-platform-python-scripts-not-so-cross-platform/) 307 | 308 | #### Configurable Post Children Callback 309 | 310 | This library provides upstream code with the ability to specify a 311 | callback function to be invoked for each directory after its children 312 | are processed. This has been used to recursively delete empty 313 | directories after traversing the file system in a more efficient 314 | manner. See the `examples/clean-empties` directory for an example of 315 | this usage. 316 | 317 | #### Configurable Error Callback 318 | 319 | This library provides upstream code with the ability to specify a 320 | callback to be invoked for errors that the operating system returns, 321 | allowing the upstream code to determine the next course of action to 322 | take, whether to halt walking the hierarchy, as it would do were no 323 | error callback provided, or skip the node that caused the error. See 324 | the `examples/walk-fast` directory for an example of this usage. 325 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | # Go 2 | # Build your Go project. 3 | # Add steps that test, save build artifacts, deploy, and more: 4 | # https://docs.microsoft.com/azure/devops/pipelines/languages/go 5 | 6 | trigger: 7 | - master 8 | 9 | variables: 10 | GOVERSION: 1.13 11 | 12 | jobs: 13 | - job: Linux 14 | pool: 15 | vmImage: 'ubuntu-latest' 16 | steps: 17 | - task: GoTool@0 18 | displayName: 'Use Go $(GOVERSION)' 19 | inputs: 20 | version: $(GOVERSION) 21 | - task: Go@0 22 | inputs: 23 | command: test 24 | arguments: -race -v ./... 25 | displayName: 'Execute Tests' 26 | 27 | - job: Mac 28 | pool: 29 | vmImage: 'macos-latest' 30 | steps: 31 | - task: GoTool@0 32 | displayName: 'Use Go $(GOVERSION)' 33 | inputs: 34 | version: $(GOVERSION) 35 | - task: Go@0 36 | inputs: 37 | command: test 38 | arguments: -race -v ./... 39 | displayName: 'Execute Tests' 40 | 41 | - job: Windows 42 | pool: 43 | vmImage: 'windows-latest' 44 | steps: 45 | - task: GoTool@0 46 | displayName: 'Use Go $(GOVERSION)' 47 | inputs: 48 | version: $(GOVERSION) 49 | - task: Go@0 50 | inputs: 51 | command: test 52 | arguments: -race -v ./... 53 | displayName: 'Execute Tests' 54 | -------------------------------------------------------------------------------- /bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # for version in v1.9.1 v1.10.0 v1.10.3 v1.10.12 v1.11.2 v1.11.3 v1.12.0 v1.13.1 v1.14.0 v1.14.1 ; do 4 | for version in v1.10.12 v1.14.1 v1.15.2 ; do 5 | echo "### $version" > $version.txt 6 | git checkout -- go.mod && git checkout $version && go test -run=NONE -bench=Benchmark2 >> $version.txt || exit 1 7 | done 8 | -------------------------------------------------------------------------------- /benchmark_test.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | ) 7 | 8 | const benchRoot = "/mnt/ram_disk/src" 9 | 10 | var scratch []byte 11 | var largeDirectory string 12 | 13 | func init() { 14 | scratch = make([]byte, MinimumScratchBufferSize) 15 | largeDirectory = filepath.Join(benchRoot, "linkedin/dashboards") 16 | } 17 | 18 | func Benchmark2ReadDirentsGodirwalk(b *testing.B) { 19 | var count int 20 | 21 | for i := 0; i < b.N; i++ { 22 | actual, err := ReadDirents(largeDirectory, scratch) 23 | if err != nil { 24 | b.Fatal(err) 25 | } 26 | count += len(actual) 27 | } 28 | 29 | _ = count 30 | } 31 | 32 | func Benchmark2ReadDirnamesGodirwalk(b *testing.B) { 33 | var count int 34 | 35 | for i := 0; i < b.N; i++ { 36 | actual, err := ReadDirnames(largeDirectory, scratch) 37 | if err != nil { 38 | b.Fatal(err) 39 | } 40 | count += len(actual) 41 | } 42 | 43 | _ = count 44 | } 45 | 46 | func Benchmark2GodirwalkSorted(b *testing.B) { 47 | for i := 0; i < b.N; i++ { 48 | var length int 49 | err := Walk(benchRoot, &Options{ 50 | Callback: func(name string, _ *Dirent) error { 51 | if name == "skip" { 52 | return filepath.SkipDir 53 | } 54 | length += len(name) 55 | return nil 56 | }, 57 | ScratchBuffer: scratch, 58 | }) 59 | if err != nil { 60 | b.Errorf("GOT: %v; WANT: nil", err) 61 | } 62 | _ = length 63 | } 64 | } 65 | 66 | func Benchmark2GodirwalkUnsorted(b *testing.B) { 67 | for i := 0; i < b.N; i++ { 68 | var length int 69 | err := Walk(benchRoot, &Options{ 70 | Callback: func(name string, _ *Dirent) error { 71 | if name == "skip" { 72 | return filepath.SkipDir 73 | } 74 | length += len(name) 75 | return nil 76 | }, 77 | ScratchBuffer: scratch, 78 | Unsorted: true, 79 | }) 80 | if err != nil { 81 | b.Errorf("GOT: %v; WANT: nil", err) 82 | } 83 | _ = length 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /debug_development.go: -------------------------------------------------------------------------------- 1 | // +build godirwalk_debug 2 | 3 | package godirwalk 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | ) 9 | 10 | // debug formats and prints arguments to stderr for development builds 11 | func debug(f string, a ...interface{}) { 12 | // fmt.Fprintf(os.Stderr, f, a...) 13 | os.Stderr.Write([]byte("godirwalk: " + fmt.Sprintf(f, a...))) 14 | } 15 | -------------------------------------------------------------------------------- /debug_release.go: -------------------------------------------------------------------------------- 1 | // +build !godirwalk_debug 2 | 3 | package godirwalk 4 | 5 | // debug is a no-op for release builds 6 | func debug(_ string, _ ...interface{}) {} 7 | -------------------------------------------------------------------------------- /dirent.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | // Dirent stores the name and file system mode type of discovered file system 9 | // entries. 10 | type Dirent struct { 11 | name string // base name of the file system entry. 12 | path string // path name of the file system entry. 13 | modeType os.FileMode // modeType is the type of file system entry. 14 | } 15 | 16 | // NewDirent returns a newly initialized Dirent structure, or an error. This 17 | // function does not follow symbolic links. 18 | // 19 | // This function is rarely used, as Dirent structures are provided by other 20 | // functions in this library that read and walk directories, but is provided, 21 | // however, for the occasion when a program needs to create a Dirent. 22 | func NewDirent(osPathname string) (*Dirent, error) { 23 | modeType, err := modeType(osPathname) 24 | if err != nil { 25 | return nil, err 26 | } 27 | return &Dirent{ 28 | name: filepath.Base(osPathname), 29 | path: filepath.Dir(osPathname), 30 | modeType: modeType, 31 | }, nil 32 | } 33 | 34 | // IsDir returns true if and only if the Dirent represents a file system 35 | // directory. Note that on some operating systems, more than one file mode bit 36 | // may be set for a node. For instance, on Windows, a symbolic link that points 37 | // to a directory will have both the directory and the symbolic link bits set. 38 | func (de Dirent) IsDir() bool { return de.modeType&os.ModeDir != 0 } 39 | 40 | // IsDirOrSymlinkToDir returns true if and only if the Dirent represents a file 41 | // system directory, or a symbolic link to a directory. Note that if the Dirent 42 | // is not a directory but is a symbolic link, this method will resolve by 43 | // sending a request to the operating system to follow the symbolic link. 44 | func (de Dirent) IsDirOrSymlinkToDir() (bool, error) { 45 | if de.IsDir() { 46 | return true, nil 47 | } 48 | if !de.IsSymlink() { 49 | return false, nil 50 | } 51 | // Does this symlink point to a directory? 52 | info, err := os.Stat(filepath.Join(de.path, de.name)) 53 | if err != nil { 54 | return false, err 55 | } 56 | return info.IsDir(), nil 57 | } 58 | 59 | // IsRegular returns true if and only if the Dirent represents a regular file. 60 | // That is, it ensures that no mode type bits are set. 61 | func (de Dirent) IsRegular() bool { return de.modeType&os.ModeType == 0 } 62 | 63 | // IsSymlink returns true if and only if the Dirent represents a file system 64 | // symbolic link. Note that on some operating systems, more than one file mode 65 | // bit may be set for a node. For instance, on Windows, a symbolic link that 66 | // points to a directory will have both the directory and the symbolic link bits 67 | // set. 68 | func (de Dirent) IsSymlink() bool { return de.modeType&os.ModeSymlink != 0 } 69 | 70 | // IsDevice returns true if and only if the Dirent represents a device file. 71 | func (de Dirent) IsDevice() bool { return de.modeType&os.ModeDevice != 0 } 72 | 73 | // ModeType returns the mode bits that specify the file system node type. We 74 | // could make our own enum-like data type for encoding the file type, but Go's 75 | // runtime already gives us architecture independent file modes, as discussed in 76 | // `os/types.go`: 77 | // 78 | // Go's runtime FileMode type has same definition on all systems, so that 79 | // information about files can be moved from one system to another portably. 80 | func (de Dirent) ModeType() os.FileMode { return de.modeType } 81 | 82 | // Name returns the base name of the file system entry. 83 | func (de Dirent) Name() string { return de.name } 84 | 85 | // reset releases memory held by entry err and name, and resets mode type to 0. 86 | func (de *Dirent) reset() { 87 | de.name = "" 88 | de.path = "" 89 | de.modeType = 0 90 | } 91 | 92 | // Dirents represents a slice of Dirent pointers, which are sortable by base 93 | // name. This type satisfies the `sort.Interface` interface. 94 | type Dirents []*Dirent 95 | 96 | // Len returns the count of Dirent structures in the slice. 97 | func (l Dirents) Len() int { return len(l) } 98 | 99 | // Less returns true if and only if the base name of the element specified by 100 | // the first index is lexicographically less than that of the second index. 101 | func (l Dirents) Less(i, j int) bool { return l[i].name < l[j].name } 102 | 103 | // Swap exchanges the two Dirent entries specified by the two provided indexes. 104 | func (l Dirents) Swap(i, j int) { l[i], l[j] = l[j], l[i] } 105 | -------------------------------------------------------------------------------- /dirent_test.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestDirent(t *testing.T) { 10 | // TODO: IsDevice() should be tested, but that would require updating 11 | // scaffolding to create a device. 12 | 13 | t.Run("file", func(t *testing.T) { 14 | de, err := NewDirent(filepath.Join(scaffolingRoot, "d0", "f1")) 15 | ensureError(t, err) 16 | 17 | if got, want := de.Name(), "f1"; got != want { 18 | t.Errorf("GOT: %v; WANT: %v", got, want) 19 | } 20 | 21 | if got, want := de.ModeType(), os.FileMode(0); got != want { 22 | t.Errorf("GOT: %v; WANT: %v", got, want) 23 | } 24 | 25 | if got, want := de.IsDir(), false; got != want { 26 | t.Errorf("GOT: %v; WANT: %v", got, want) 27 | } 28 | 29 | got, err := de.IsDirOrSymlinkToDir() 30 | ensureError(t, err) 31 | 32 | if want := false; got != want { 33 | t.Errorf("GOT: %v; WANT: %v", got, want) 34 | } 35 | 36 | if got, want := de.IsRegular(), true; got != want { 37 | t.Errorf("GOT: %v; WANT: %v", got, want) 38 | } 39 | 40 | if got, want := de.IsSymlink(), false; got != want { 41 | t.Errorf("GOT: %v; WANT: %v", got, want) 42 | } 43 | }) 44 | 45 | t.Run("directory", func(t *testing.T) { 46 | de, err := NewDirent(filepath.Join(scaffolingRoot, "d0")) 47 | ensureError(t, err) 48 | 49 | if got, want := de.Name(), "d0"; got != want { 50 | t.Errorf("GOT: %v; WANT: %v", got, want) 51 | } 52 | 53 | if got, want := de.ModeType(), os.ModeDir; got != want { 54 | t.Errorf("GOT: %v; WANT: %v", got, want) 55 | } 56 | 57 | if got, want := de.IsDir(), true; got != want { 58 | t.Errorf("GOT: %v; WANT: %v", got, want) 59 | } 60 | 61 | got, err := de.IsDirOrSymlinkToDir() 62 | ensureError(t, err) 63 | 64 | if want := true; got != want { 65 | t.Errorf("GOT: %v; WANT: %v", got, want) 66 | } 67 | 68 | if got, want := de.IsRegular(), false; got != want { 69 | t.Errorf("GOT: %v; WANT: %v", got, want) 70 | } 71 | 72 | if got, want := de.IsSymlink(), false; got != want { 73 | t.Errorf("GOT: %v; WANT: %v", got, want) 74 | } 75 | }) 76 | 77 | t.Run("symlink", func(t *testing.T) { 78 | t.Run("to file", func(t *testing.T) { 79 | de, err := NewDirent(filepath.Join(scaffolingRoot, "d0", "symlinks", "toF1")) 80 | ensureError(t, err) 81 | 82 | if got, want := de.Name(), "toF1"; got != want { 83 | t.Errorf("GOT: %v; WANT: %v", got, want) 84 | } 85 | 86 | if got, want := de.ModeType(), os.ModeSymlink; got != want { 87 | t.Errorf("GOT: %v; WANT: %v", got, want) 88 | } 89 | 90 | if got, want := de.IsDir(), false; got != want { 91 | t.Errorf("GOT: %v; WANT: %v", got, want) 92 | } 93 | 94 | got, err := de.IsDirOrSymlinkToDir() 95 | ensureError(t, err) 96 | 97 | if want := false; got != want { 98 | t.Errorf("GOT: %v; WANT: %v", got, want) 99 | } 100 | 101 | if got, want := de.IsRegular(), false; got != want { 102 | t.Errorf("GOT: %v; WANT: %v", got, want) 103 | } 104 | 105 | if got, want := de.IsSymlink(), true; got != want { 106 | t.Errorf("GOT: %v; WANT: %v", got, want) 107 | } 108 | }) 109 | 110 | t.Run("to directory", func(t *testing.T) { 111 | de, err := NewDirent(filepath.Join(scaffolingRoot, "d0", "symlinks", "toD1")) 112 | ensureError(t, err) 113 | 114 | if got, want := de.Name(), "toD1"; got != want { 115 | t.Errorf("GOT: %v; WANT: %v", got, want) 116 | } 117 | 118 | if got, want := de.ModeType(), os.ModeSymlink; got != want { 119 | t.Errorf("GOT: %v; WANT: %v", got, want) 120 | } 121 | 122 | if got, want := de.IsDir(), false; got != want { 123 | t.Errorf("GOT: %v; WANT: %v", got, want) 124 | } 125 | 126 | got, err := de.IsDirOrSymlinkToDir() 127 | ensureError(t, err) 128 | 129 | if want := true; got != want { 130 | t.Errorf("GOT: %v; WANT: %v", got, want) 131 | } 132 | 133 | if got, want := de.IsRegular(), false; got != want { 134 | t.Errorf("GOT: %v; WANT: %v", got, want) 135 | } 136 | 137 | if got, want := de.IsSymlink(), true; got != want { 138 | t.Errorf("GOT: %v; WANT: %v", got, want) 139 | } 140 | }) 141 | }) 142 | } 143 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package godirwalk provides functions to read and traverse directory trees. 3 | 4 | In short, why do I use this library? 5 | 6 | * It's faster than `filepath.Walk`. 7 | 8 | * It's more correct on Windows than `filepath.Walk`. 9 | 10 | * It's more easy to use than `filepath.Walk`. 11 | 12 | * It's more flexible than `filepath.Walk`. 13 | 14 | USAGE 15 | 16 | This library will normalize the provided top level directory name based on the 17 | os-specific path separator by calling `filepath.Clean` on its first 18 | argument. However it always provides the pathname created by using the correct 19 | os-specific path separator when invoking the provided callback function. 20 | 21 | dirname := "some/directory/root" 22 | err := godirwalk.Walk(dirname, &godirwalk.Options{ 23 | Callback: func(osPathname string, de *godirwalk.Dirent) error { 24 | fmt.Printf("%s %s\n", de.ModeType(), osPathname) 25 | return nil 26 | }, 27 | }) 28 | 29 | This library not only provides functions for traversing a file system directory 30 | tree, but also for obtaining a list of immediate descendants of a particular 31 | directory, typically much more quickly than using `os.ReadDir` or 32 | `os.ReadDirnames`. 33 | 34 | scratchBuffer := make([]byte, godirwalk.MinimumScratchBufferSize) 35 | 36 | names, err := godirwalk.ReadDirnames("some/directory", scratchBuffer) 37 | // ... 38 | 39 | entries, err := godirwalk.ReadDirents("another/directory", scratchBuffer) 40 | // ... 41 | */ 42 | package godirwalk 43 | -------------------------------------------------------------------------------- /ensure_test.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "sort" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func ensureError(tb testing.TB, err error, contains ...string) { 12 | tb.Helper() 13 | if len(contains) == 0 || (len(contains) == 1 && contains[0] == "") { 14 | if err != nil { 15 | tb.Fatalf("GOT: %v; WANT: %v", err, contains) 16 | } 17 | } else if err == nil { 18 | tb.Errorf("GOT: %v; WANT: %v", err, contains) 19 | } else { 20 | for _, stub := range contains { 21 | if stub != "" && !strings.Contains(err.Error(), stub) { 22 | tb.Errorf("GOT: %v; WANT: %q", err, stub) 23 | } 24 | } 25 | } 26 | } 27 | 28 | func ensureStringSlicesMatch(tb testing.TB, actual, expected []string) { 29 | tb.Helper() 30 | 31 | results := make(map[string]int) 32 | 33 | for _, s := range actual { 34 | results[s] = -1 35 | } 36 | for _, s := range expected { 37 | results[s]++ 38 | } 39 | 40 | keys := make([]string, 0, len(results)) 41 | for k := range results { 42 | keys = append(keys, k) 43 | } 44 | sort.Strings(keys) 45 | 46 | for _, s := range keys { 47 | v, ok := results[s] 48 | if !ok { 49 | panic(fmt.Errorf("cannot find key: %s", s)) // panic because this function is broken 50 | } 51 | switch v { 52 | case -1: 53 | tb.Errorf("GOT: %q (extra)", s) 54 | case 0: 55 | // both slices have this key 56 | case 1: 57 | tb.Errorf("WANT: %q (missing)", s) 58 | default: 59 | panic(fmt.Errorf("key has invalid value: %s: %d", s, v)) // panic because this function is broken 60 | } 61 | } 62 | } 63 | 64 | func ensureDirentsMatch(tb testing.TB, actual, expected Dirents) { 65 | tb.Helper() 66 | 67 | sort.Sort(actual) 68 | sort.Sort(expected) 69 | 70 | al := len(actual) 71 | el := len(expected) 72 | var ai, ei int 73 | 74 | for ai < al || ei < el { 75 | if ai == al { 76 | tb.Errorf("GOT: %s %s (extra)", expected[ei].Name(), expected[ei].ModeType()) 77 | ei++ 78 | } else if ei == el { 79 | tb.Errorf("WANT: %s %s (missing)", actual[ai].Name(), actual[ai].ModeType()) 80 | ai++ 81 | } else { 82 | epn := filepath.Join(expected[ei].path, expected[ei].Name()) 83 | apn := filepath.Join(actual[ai].path, actual[ai].Name()) 84 | 85 | if apn < epn { 86 | tb.Errorf("GOT: %s %s (extra)", apn, actual[ai].ModeType()) 87 | ai++ 88 | } else if epn < apn { 89 | tb.Errorf("WANT: %s %s (missing)", epn, expected[ei].ModeType()) 90 | ei++ 91 | } else { 92 | // names match; check mode types 93 | if got, want := actual[ai].ModeType(), expected[ei].ModeType(); got != want { 94 | tb.Errorf("GOT: %v; WANT: %v", actual[ai].ModeType(), expected[ei].ModeType()) 95 | } 96 | ai++ 97 | ei++ 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /examples/find-fast/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/karrick/godirwalk/examples/find-fast 2 | 3 | replace github.com/karrick/godirwalk => ../../ 4 | 5 | require ( 6 | github.com/karrick/godirwalk v1.13.5 7 | github.com/karrick/golf v1.4.0 8 | github.com/mattn/go-isatty v0.0.11 9 | golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect 10 | ) 11 | 12 | go 1.13 13 | -------------------------------------------------------------------------------- /examples/find-fast/go.sum: -------------------------------------------------------------------------------- 1 | github.com/karrick/golf v1.4.0 h1:9i9HnUh7uCyUFJhIqg311HBibw4f2pbGldi0ZM2FhaQ= 2 | github.com/karrick/golf v1.4.0/go.mod h1:qGN0IhcEL+IEgCXp00RvH32UP59vtwc8w5YcIdArNRk= 3 | github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= 4 | github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= 5 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 6 | golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= 7 | golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 8 | -------------------------------------------------------------------------------- /examples/find-fast/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * find-fast 3 | * 4 | * Walks a file system hierarchy using this library. 5 | */ 6 | package main 7 | 8 | import ( 9 | "fmt" 10 | "os" 11 | "path/filepath" 12 | "regexp" 13 | "strings" 14 | 15 | "github.com/karrick/godirwalk" 16 | "github.com/karrick/golf" 17 | "github.com/mattn/go-isatty" 18 | ) 19 | 20 | var NoColor = os.Getenv("TERM") == "dumb" || !(isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())) 21 | 22 | func main() { 23 | optQuiet := golf.Bool("quiet", false, "Do not print intermediate errors to stderr.") 24 | optRegex := golf.String("regex", "", "Do not print unless full path matches regex.") 25 | optSkip := golf.String("skip", "", "Skip and do not descend into entries with this substring in the pathname") 26 | golf.Parse() 27 | 28 | programName, err := os.Executable() 29 | if err != nil { 30 | programName = os.Args[0] 31 | } 32 | programName = filepath.Base(programName) 33 | 34 | var nameRE *regexp.Regexp 35 | if *optRegex != "" { 36 | nameRE, err = regexp.Compile(*optRegex) 37 | if err != nil { 38 | fmt.Fprintf(os.Stderr, "%s: invalid regex pattern: %s\n", programName, err) 39 | os.Exit(2) 40 | } 41 | } 42 | 43 | var buf []byte // only used when color output 44 | 45 | options := &godirwalk.Options{ 46 | ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { 47 | if !*optQuiet { 48 | fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) 49 | } 50 | return godirwalk.SkipNode 51 | }, 52 | Unsorted: true, 53 | } 54 | 55 | switch { 56 | case nameRE == nil: 57 | // When no name pattern provided, print everything. 58 | options.Callback = func(osPathname string, de *godirwalk.Dirent) error { 59 | if *optSkip != "" && strings.Contains(osPathname, *optSkip) { 60 | if !*optQuiet { 61 | fmt.Fprintf(os.Stderr, "%s: %s (skipping)\n", programName, osPathname) 62 | } 63 | return godirwalk.SkipThis 64 | } 65 | _, err := fmt.Println(osPathname) 66 | return err 67 | } 68 | case NoColor: 69 | // Name pattern was provided, but color not permitted. 70 | options.Callback = func(osPathname string, _ *godirwalk.Dirent) error { 71 | if *optSkip != "" && strings.Contains(osPathname, *optSkip) { 72 | if !*optQuiet { 73 | fmt.Fprintf(os.Stderr, "%s: %s (skipping)\n", programName, osPathname) 74 | } 75 | return godirwalk.SkipThis 76 | } 77 | var err error 78 | if nameRE.FindString(osPathname) != "" { 79 | _, err = fmt.Println(osPathname) 80 | } 81 | return err 82 | } 83 | default: 84 | // Name pattern provided, and color is permitted. 85 | buf = append(buf, "\033[22m"...) // very first print should set normal intensity 86 | 87 | options.Callback = func(osPathname string, _ *godirwalk.Dirent) error { 88 | if *optSkip != "" && strings.Contains(osPathname, *optSkip) { 89 | if !*optQuiet { 90 | fmt.Fprintf(os.Stderr, "%s: %s (skipping)\n", programName, osPathname) 91 | } 92 | return godirwalk.SkipThis 93 | } 94 | matches := nameRE.FindAllStringSubmatchIndex(osPathname, -1) 95 | if len(matches) == 0 { 96 | return nil // entry does not match pattern 97 | } 98 | 99 | var prev int 100 | for _, tuple := range matches { 101 | buf = append(buf, osPathname[prev:tuple[0]]...) // print text before match 102 | buf = append(buf, "\033[1m"...) // bold intensity 103 | buf = append(buf, osPathname[tuple[0]:tuple[1]]...) // print match 104 | buf = append(buf, "\033[22m"...) // normal intensity 105 | prev = tuple[1] 106 | } 107 | 108 | buf = append(buf, osPathname[prev:]...) // print remaining text after final match 109 | _, err := os.Stdout.Write(append(buf, '\n')) // don't forget newline 110 | buf = buf[:0] // reset buffer for next string 111 | return err 112 | } 113 | } 114 | 115 | dirname := "." 116 | if golf.NArg() > 0 { 117 | dirname = golf.Arg(0) 118 | } 119 | 120 | if err = godirwalk.Walk(dirname, options); err != nil { 121 | fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) 122 | os.Exit(1) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /examples/remove-empty-directories/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * remove-empty-directories 3 | * 4 | * Walks a file system hierarchy and removes all directories with no children. 5 | */ 6 | package main 7 | 8 | import ( 9 | "fmt" 10 | "os" 11 | "path/filepath" 12 | 13 | "github.com/karrick/godirwalk" 14 | ) 15 | 16 | func main() { 17 | if len(os.Args) < 2 { 18 | fmt.Fprintf(os.Stderr, "usage: %s dir1 [dir2 [dir3...]]\n", filepath.Base(os.Args[0])) 19 | os.Exit(2) 20 | } 21 | 22 | var count, total int 23 | var err error 24 | 25 | for _, arg := range os.Args[1:] { 26 | count, err = pruneEmptyDirectories(arg) 27 | total += count 28 | if err != nil { 29 | break 30 | } 31 | } 32 | 33 | fmt.Fprintf(os.Stderr, "Removed %d empty directories\n", total) 34 | if err != nil { 35 | fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) 36 | os.Exit(1) 37 | } 38 | } 39 | 40 | func pruneEmptyDirectories(osDirname string) (int, error) { 41 | var count int 42 | 43 | err := godirwalk.Walk(osDirname, &godirwalk.Options{ 44 | Unsorted: true, 45 | Callback: func(_ string, _ *godirwalk.Dirent) error { 46 | // no-op while diving in; all the fun happens in PostChildrenCallback 47 | return nil 48 | }, 49 | PostChildrenCallback: func(osPathname string, _ *godirwalk.Dirent) error { 50 | s, err := godirwalk.NewScanner(osPathname) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | // Attempt to read only the first directory entry. Remember that 56 | // Scan skips both "." and ".." entries. 57 | hasAtLeastOneChild := s.Scan() 58 | 59 | // If error reading from directory, wrap up and return. 60 | if err := s.Err(); err != nil { 61 | return err 62 | } 63 | 64 | if hasAtLeastOneChild { 65 | return nil // do not remove directory with at least one child 66 | } 67 | if osPathname == osDirname { 68 | return nil // do not remove directory that was provided top-level directory 69 | } 70 | 71 | err = os.Remove(osPathname) 72 | if err == nil { 73 | count++ 74 | } 75 | return err 76 | }, 77 | }) 78 | 79 | return count, err 80 | } 81 | -------------------------------------------------------------------------------- /examples/scanner/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/karrick/godirwalk" 10 | ) 11 | 12 | func main() { 13 | dirname := "." 14 | if flag.NArg() > 0 { 15 | dirname = flag.Arg(0) 16 | } 17 | 18 | scanner, err := godirwalk.NewScanner(dirname) 19 | if err != nil { 20 | fatal("cannot scan directory: %s", err) 21 | } 22 | 23 | for scanner.Scan() { 24 | dirent, err := scanner.Dirent() 25 | if err != nil { 26 | warning("cannot get dirent: %s", err) 27 | continue 28 | } 29 | name := dirent.Name() 30 | if name == "break" { 31 | break 32 | } 33 | if name == "continue" { 34 | continue 35 | } 36 | fmt.Printf("%v %v\n", dirent.ModeType(), name) 37 | } 38 | if err := scanner.Err(); err != nil { 39 | fatal("cannot scan directory: %s", err) 40 | } 41 | } 42 | 43 | var ( 44 | optQuiet = flag.Bool("quiet", false, "Elide printing of non-critical error messages.") 45 | programName string 46 | ) 47 | 48 | func init() { 49 | var err error 50 | if programName, err = os.Executable(); err != nil { 51 | programName = os.Args[0] 52 | } 53 | programName = filepath.Base(programName) 54 | flag.Parse() 55 | } 56 | 57 | func stderr(f string, args ...interface{}) { 58 | fmt.Fprintf(os.Stderr, programName+": "+fmt.Sprintf(f, args...)+"\n") 59 | } 60 | 61 | func fatal(f string, args ...interface{}) { 62 | stderr(f, args...) 63 | os.Exit(1) 64 | } 65 | 66 | func warning(f string, args ...interface{}) { 67 | if !*optQuiet { 68 | stderr(f, args...) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /examples/sizes/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * sizes 3 | * 4 | * Walks a file system hierarchy and prints sizes of file system objects, 5 | * recursively printing sizes of directories. 6 | */ 7 | package main 8 | 9 | import ( 10 | "fmt" 11 | "os" 12 | "path/filepath" 13 | 14 | "github.com/karrick/godirwalk" 15 | ) 16 | 17 | var progname = filepath.Base(os.Args[0]) 18 | 19 | func main() { 20 | if len(os.Args) < 2 { 21 | fmt.Fprintf(os.Stderr, "usage: %s dir1 [dir2 [dir3...]]\n", progname) 22 | os.Exit(2) 23 | } 24 | 25 | for _, arg := range os.Args[1:] { 26 | if err := sizes(arg); err != nil { 27 | fmt.Fprintf(os.Stderr, "%s: %s\n", progname, err) 28 | } 29 | } 30 | } 31 | 32 | func sizes(osDirname string) error { 33 | sizes := newSizesStack() 34 | 35 | return godirwalk.Walk(osDirname, &godirwalk.Options{ 36 | Callback: func(osPathname string, de *godirwalk.Dirent) error { 37 | if de.IsDir() { 38 | sizes.EnterDirectory() 39 | return nil 40 | } 41 | 42 | st, err := os.Stat(osPathname) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | size := st.Size() 48 | sizes.Accumulate(size) 49 | 50 | _, err = fmt.Printf("%s % 12d %s\n", st.Mode(), size, osPathname) 51 | return err 52 | }, 53 | ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { 54 | fmt.Fprintf(os.Stderr, "%s: %s\n", progname, err) 55 | return godirwalk.SkipNode 56 | }, 57 | PostChildrenCallback: func(osPathname string, de *godirwalk.Dirent) error { 58 | size := sizes.LeaveDirectory() 59 | sizes.Accumulate(size) // add this directory's size to parent directory. 60 | 61 | st, err := os.Stat(osPathname) 62 | 63 | switch err { 64 | case nil: 65 | _, err = fmt.Printf("%s % 12d %s\n", st.Mode(), size, osPathname) 66 | default: 67 | // ignore the error and just show the mode type 68 | _, err = fmt.Printf("%s % 12d %s\n", de.ModeType(), size, osPathname) 69 | } 70 | return err 71 | }, 72 | }) 73 | } 74 | 75 | // sizesStack encapsulates operations on stack of directory sizes, with similar 76 | // but slightly modified LIFO semantics to push and pop on a regular stack. 77 | type sizesStack struct { 78 | sizes []int64 // stack of sizes 79 | top int // index of top of stack 80 | } 81 | 82 | func newSizesStack() *sizesStack { 83 | // Initialize with dummy value at top of stack to eliminate special cases. 84 | return &sizesStack{sizes: make([]int64, 1, 32)} 85 | } 86 | 87 | func (s *sizesStack) EnterDirectory() { 88 | s.sizes = append(s.sizes, 0) 89 | s.top++ 90 | } 91 | 92 | func (s *sizesStack) LeaveDirectory() (i int64) { 93 | i, s.sizes = s.sizes[s.top], s.sizes[:s.top] 94 | s.top-- 95 | return i 96 | } 97 | 98 | func (s *sizesStack) Accumulate(i int64) { 99 | s.sizes[s.top] += i 100 | } 101 | -------------------------------------------------------------------------------- /examples/walk-fast/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * walk-fast 3 | * 4 | * Walks a file system hierarchy using this library. 5 | */ 6 | package main 7 | 8 | import ( 9 | "flag" 10 | "fmt" 11 | "os" 12 | 13 | "github.com/karrick/godirwalk" 14 | ) 15 | 16 | func main() { 17 | optVerbose := flag.Bool("verbose", false, "Print file system entries.") 18 | flag.Parse() 19 | 20 | dirname := "." 21 | if flag.NArg() > 0 { 22 | dirname = flag.Arg(0) 23 | } 24 | 25 | err := godirwalk.Walk(dirname, &godirwalk.Options{ 26 | Callback: func(osPathname string, de *godirwalk.Dirent) error { 27 | if *optVerbose { 28 | fmt.Printf("%s %s\n", de.ModeType(), osPathname) 29 | } 30 | return nil 31 | }, 32 | ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { 33 | if *optVerbose { 34 | fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) 35 | } 36 | 37 | // For the purposes of this example, a simple SkipNode will suffice, 38 | // although in reality perhaps additional logic might be called for. 39 | return godirwalk.SkipNode 40 | }, 41 | Unsorted: true, // set true for faster yet non-deterministic enumeration (see godoc) 42 | }) 43 | if err != nil { 44 | fmt.Fprintf(os.Stderr, "%s\n", err) 45 | os.Exit(1) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /examples/walk-stdlib/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * walk-fast 3 | * 4 | * Walks a file system hierarchy using the standard library. 5 | */ 6 | package main 7 | 8 | import ( 9 | "flag" 10 | "fmt" 11 | "os" 12 | "path/filepath" 13 | ) 14 | 15 | func main() { 16 | optVerbose := flag.Bool("verbose", false, "Print file system entries.") 17 | flag.Parse() 18 | 19 | dirname := "." 20 | if flag.NArg() > 0 { 21 | dirname = flag.Arg(0) 22 | } 23 | 24 | err := filepath.Walk(dirname, func(osPathname string, info os.FileInfo, err error) error { 25 | if err != nil { 26 | return err 27 | } 28 | if *optVerbose { 29 | fmt.Printf("%s %s\n", info.Mode(), osPathname) 30 | } 31 | return nil 32 | }) 33 | if err != nil { 34 | fmt.Fprintf(os.Stderr, "%s\n", err) 35 | os.Exit(1) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/karrick/godirwalk 2 | 3 | go 1.13 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karrick/godirwalk/9a7752c108e7ea76255201b9f47bd4d4d2df868e/go.sum -------------------------------------------------------------------------------- /inoWithFileno.go: -------------------------------------------------------------------------------- 1 | // +build dragonfly freebsd openbsd netbsd 2 | 3 | package godirwalk 4 | 5 | import "syscall" 6 | 7 | func inoFromDirent(de *syscall.Dirent) uint64 { 8 | return uint64(de.Fileno) 9 | } 10 | -------------------------------------------------------------------------------- /inoWithIno.go: -------------------------------------------------------------------------------- 1 | // +build aix darwin linux nacl solaris 2 | 3 | package godirwalk 4 | 5 | import "syscall" 6 | 7 | func inoFromDirent(de *syscall.Dirent) uint64 { 8 | return uint64(de.Ino) 9 | } 10 | -------------------------------------------------------------------------------- /modeType.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // modeType returns the mode type of the file system entry identified by 8 | // osPathname by calling os.LStat function, to intentionally not follow symbolic 9 | // links. 10 | // 11 | // Even though os.LStat provides all file mode bits, we want to ensure same 12 | // values returned to caller regardless of whether we obtained file mode bits 13 | // from syscall or stat call. Therefore mask out the additional file mode bits 14 | // that are provided by stat but not by the syscall, so users can rely on their 15 | // values. 16 | func modeType(osPathname string) (os.FileMode, error) { 17 | fi, err := os.Lstat(osPathname) 18 | if err == nil { 19 | return fi.Mode() & os.ModeType, nil 20 | } 21 | return 0, err 22 | } 23 | -------------------------------------------------------------------------------- /modeTypeWithType.go: -------------------------------------------------------------------------------- 1 | // +build darwin dragonfly freebsd linux netbsd openbsd 2 | 3 | package godirwalk 4 | 5 | import ( 6 | "os" 7 | "path/filepath" 8 | "syscall" 9 | ) 10 | 11 | // modeTypeFromDirent converts a syscall defined constant, which is in purview 12 | // of OS, to a constant defined by Go, assumed by this project to be stable. 13 | // 14 | // When the syscall constant is not recognized, this function falls back to a 15 | // Stat on the file system. 16 | func modeTypeFromDirent(de *syscall.Dirent, osDirname, osBasename string) (os.FileMode, error) { 17 | switch de.Type { 18 | case syscall.DT_REG: 19 | return 0, nil 20 | case syscall.DT_DIR: 21 | return os.ModeDir, nil 22 | case syscall.DT_LNK: 23 | return os.ModeSymlink, nil 24 | case syscall.DT_CHR: 25 | return os.ModeDevice | os.ModeCharDevice, nil 26 | case syscall.DT_BLK: 27 | return os.ModeDevice, nil 28 | case syscall.DT_FIFO: 29 | return os.ModeNamedPipe, nil 30 | case syscall.DT_SOCK: 31 | return os.ModeSocket, nil 32 | default: 33 | // If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), then 34 | // resolve actual mode by reading file information. 35 | return modeType(filepath.Join(osDirname, osBasename)) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /modeTypeWithoutType.go: -------------------------------------------------------------------------------- 1 | // +build aix js nacl solaris 2 | 3 | package godirwalk 4 | 5 | import ( 6 | "os" 7 | "path/filepath" 8 | "syscall" 9 | ) 10 | 11 | // modeTypeFromDirent converts a syscall defined constant, which is in purview 12 | // of OS, to a constant defined by Go, assumed by this project to be stable. 13 | // 14 | // Because some operating system syscall.Dirent structures do not include a Type 15 | // field, fall back on Stat of the file system. 16 | func modeTypeFromDirent(_ *syscall.Dirent, osDirname, osBasename string) (os.FileMode, error) { 17 | return modeType(filepath.Join(osDirname, osBasename)) 18 | } 19 | -------------------------------------------------------------------------------- /nameWithNamlen.go: -------------------------------------------------------------------------------- 1 | // +build aix darwin dragonfly freebsd netbsd openbsd 2 | 3 | package godirwalk 4 | 5 | import ( 6 | "reflect" 7 | "syscall" 8 | "unsafe" 9 | ) 10 | 11 | func nameFromDirent(de *syscall.Dirent) []byte { 12 | // Because this GOOS' syscall.Dirent provides a Namlen field that says how 13 | // long the name is, this function does not need to search for the NULL 14 | // byte. 15 | ml := int(de.Namlen) 16 | 17 | // Convert syscall.Dirent.Name, which is array of int8, to []byte, by 18 | // overwriting Cap, Len, and Data slice header fields to values from 19 | // syscall.Dirent fields. Setting the Cap, Len, and Data field values for 20 | // the slice header modifies what the slice header points to, and in this 21 | // case, the name buffer. 22 | var name []byte 23 | sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) 24 | sh.Cap = ml 25 | sh.Len = ml 26 | sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) 27 | 28 | return name 29 | } 30 | -------------------------------------------------------------------------------- /nameWithoutNamlen.go: -------------------------------------------------------------------------------- 1 | // +build nacl linux js solaris 2 | 3 | package godirwalk 4 | 5 | import ( 6 | "bytes" 7 | "reflect" 8 | "syscall" 9 | "unsafe" 10 | ) 11 | 12 | // nameOffset is a compile time constant 13 | const nameOffset = int(unsafe.Offsetof(syscall.Dirent{}.Name)) 14 | 15 | func nameFromDirent(de *syscall.Dirent) (name []byte) { 16 | // Because this GOOS' syscall.Dirent does not provide a field that specifies 17 | // the name length, this function must first calculate the max possible name 18 | // length, and then search for the NULL byte. 19 | ml := int(de.Reclen) - nameOffset 20 | 21 | // Convert syscall.Dirent.Name, which is array of int8, to []byte, by 22 | // overwriting Cap, Len, and Data slice header fields to the max possible 23 | // name length computed above, and finding the terminating NULL byte. 24 | sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) 25 | sh.Cap = ml 26 | sh.Len = ml 27 | sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) 28 | 29 | if index := bytes.IndexByte(name, 0); index >= 0 { 30 | // Found NULL byte; set slice's cap and len accordingly. 31 | sh.Cap = index 32 | sh.Len = index 33 | return 34 | } 35 | 36 | // NOTE: This branch is not expected, but included for defensive 37 | // programming, and provides a hard stop on the name based on the structure 38 | // field array size. 39 | sh.Cap = len(de.Name) 40 | sh.Len = sh.Cap 41 | return 42 | } 43 | -------------------------------------------------------------------------------- /readdir.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | // ReadDirents returns a sortable slice of pointers to Dirent structures, each 4 | // representing the file system name and mode type for one of the immediate 5 | // descendant of the specified directory. If the specified directory is a 6 | // symbolic link, it will be resolved. 7 | // 8 | // If an optional scratch buffer is provided that is at least one page of 9 | // memory, it will be used when reading directory entries from the file 10 | // system. If you plan on calling this function in a loop, you will have 11 | // significantly better performance if you allocate a scratch buffer and use it 12 | // each time you call this function. 13 | // 14 | // children, err := godirwalk.ReadDirents(osDirname, nil) 15 | // if err != nil { 16 | // return nil, errors.Wrap(err, "cannot get list of directory children") 17 | // } 18 | // sort.Sort(children) 19 | // for _, child := range children { 20 | // fmt.Printf("%s %s\n", child.ModeType, child.Name) 21 | // } 22 | func ReadDirents(osDirname string, scratchBuffer []byte) (Dirents, error) { 23 | return readDirents(osDirname, scratchBuffer) 24 | } 25 | 26 | // ReadDirnames returns a slice of strings, representing the immediate 27 | // descendants of the specified directory. If the specified directory is a 28 | // symbolic link, it will be resolved. 29 | // 30 | // If an optional scratch buffer is provided that is at least one page of 31 | // memory, it will be used when reading directory entries from the file 32 | // system. If you plan on calling this function in a loop, you will have 33 | // significantly better performance if you allocate a scratch buffer and use it 34 | // each time you call this function. 35 | // 36 | // Note that this function, depending on operating system, may or may not invoke 37 | // the ReadDirents function, in order to prepare the list of immediate 38 | // descendants. Therefore, if your program needs both the names and the file 39 | // system mode types of descendants, it will always be faster to invoke 40 | // ReadDirents directly, rather than calling this function, then looping over 41 | // the results and calling os.Stat or os.LStat for each entry. 42 | // 43 | // children, err := godirwalk.ReadDirnames(osDirname, nil) 44 | // if err != nil { 45 | // return nil, errors.Wrap(err, "cannot get list of directory children") 46 | // } 47 | // sort.Strings(children) 48 | // for _, child := range children { 49 | // fmt.Printf("%s\n", child) 50 | // } 51 | func ReadDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { 52 | return readDirnames(osDirname, scratchBuffer) 53 | } 54 | -------------------------------------------------------------------------------- /readdir_test.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestReadDirents(t *testing.T) { 10 | t.Run("without symlinks", func(t *testing.T) { 11 | testroot := filepath.Join(scaffolingRoot, "d0") 12 | 13 | actual, err := ReadDirents(testroot, nil) 14 | ensureError(t, err) 15 | 16 | expected := Dirents{ 17 | &Dirent{ 18 | name: maxName, 19 | path: testroot, 20 | modeType: os.FileMode(0), 21 | }, 22 | &Dirent{ 23 | name: "d1", 24 | path: testroot, 25 | modeType: os.ModeDir, 26 | }, 27 | &Dirent{ 28 | name: "f1", 29 | path: testroot, 30 | modeType: os.FileMode(0), 31 | }, 32 | &Dirent{ 33 | name: "skips", 34 | path: testroot, 35 | modeType: os.ModeDir, 36 | }, 37 | &Dirent{ 38 | name: "symlinks", 39 | path: testroot, 40 | modeType: os.ModeDir, 41 | }, 42 | } 43 | 44 | ensureDirentsMatch(t, actual, expected) 45 | }) 46 | 47 | t.Run("with symlinks", func(t *testing.T) { 48 | testroot := filepath.Join(scaffolingRoot, "d0/symlinks") 49 | 50 | actual, err := ReadDirents(testroot, nil) 51 | ensureError(t, err) 52 | 53 | // Because some platforms set multiple mode type bits, when we create 54 | // the expected slice, we need to ensure the mode types are set 55 | // appropriately for this platform. We have another test function to 56 | // ensure NewDirent does this correctly, so let's call NewDirent for 57 | // each of the expected children entries. 58 | var expected Dirents 59 | for _, child := range []string{"nothing", "toAbs", "toD1", "toF1", "d4"} { 60 | de, err := NewDirent(filepath.Join(testroot, child)) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | expected = append(expected, de) 65 | } 66 | 67 | ensureDirentsMatch(t, actual, expected) 68 | }) 69 | } 70 | 71 | func TestReadDirnames(t *testing.T) { 72 | actual, err := ReadDirnames(filepath.Join(scaffolingRoot, "d0"), nil) 73 | ensureError(t, err) 74 | expected := []string{maxName, "d1", "f1", "skips", "symlinks"} 75 | ensureStringSlicesMatch(t, actual, expected) 76 | } 77 | 78 | func BenchmarkReadDirnamesStandardLibrary(b *testing.B) { 79 | if testing.Short() { 80 | b.Skip("Skipping benchmark using user's Go source directory") 81 | } 82 | 83 | f := func(osDirname string) ([]string, error) { 84 | dh, err := os.Open(osDirname) 85 | if err != nil { 86 | return nil, err 87 | } 88 | return dh.Readdirnames(-1) 89 | } 90 | 91 | var count int 92 | 93 | for i := 0; i < b.N; i++ { 94 | actual, err := f(goPrefix) 95 | if err != nil { 96 | b.Fatal(err) 97 | } 98 | count = len(actual) 99 | } 100 | _ = count 101 | } 102 | 103 | func BenchmarkReadDirnamesGodirwalk(b *testing.B) { 104 | if testing.Short() { 105 | b.Skip("Skipping benchmark using user's Go source directory") 106 | } 107 | 108 | var count int 109 | 110 | for i := 0; i < b.N; i++ { 111 | actual, err := ReadDirnames(goPrefix, nil) 112 | if err != nil { 113 | b.Fatal(err) 114 | } 115 | count = len(actual) 116 | } 117 | _ = count 118 | } 119 | -------------------------------------------------------------------------------- /readdir_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package godirwalk 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | "unsafe" 9 | ) 10 | 11 | // MinimumScratchBufferSize specifies the minimum size of the scratch buffer 12 | // that ReadDirents, ReadDirnames, Scanner, and Walk will use when reading file 13 | // entries from the operating system. During program startup it is initialized 14 | // to the result from calling `os.Getpagesize()` for non Windows environments, 15 | // and 0 for Windows. 16 | var MinimumScratchBufferSize = os.Getpagesize() 17 | 18 | func newScratchBuffer() []byte { return make([]byte, MinimumScratchBufferSize) } 19 | 20 | func readDirents(osDirname string, scratchBuffer []byte) ([]*Dirent, error) { 21 | var entries []*Dirent 22 | var workBuffer []byte 23 | 24 | dh, err := os.Open(osDirname) 25 | if err != nil { 26 | return nil, err 27 | } 28 | fd := int(dh.Fd()) 29 | 30 | if len(scratchBuffer) < MinimumScratchBufferSize { 31 | scratchBuffer = newScratchBuffer() 32 | } 33 | 34 | var sde syscall.Dirent 35 | for { 36 | if len(workBuffer) == 0 { 37 | n, err := syscall.ReadDirent(fd, scratchBuffer) 38 | // n, err := unix.ReadDirent(fd, scratchBuffer) 39 | if err != nil { 40 | if err == syscall.EINTR /* || err == unix.EINTR */ { 41 | continue 42 | } 43 | _ = dh.Close() 44 | return nil, err 45 | } 46 | if n <= 0 { // end of directory: normal exit 47 | if err = dh.Close(); err != nil { 48 | return nil, err 49 | } 50 | return entries, nil 51 | } 52 | workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read 53 | } 54 | 55 | copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(&sde))[:], workBuffer) 56 | workBuffer = workBuffer[reclen(&sde):] // advance buffer for next iteration through loop 57 | 58 | if inoFromDirent(&sde) == 0 { 59 | continue // inode set to 0 indicates an entry that was marked as deleted 60 | } 61 | 62 | nameSlice := nameFromDirent(&sde) 63 | nameLength := len(nameSlice) 64 | 65 | if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { 66 | continue 67 | } 68 | 69 | childName := string(nameSlice) 70 | mt, err := modeTypeFromDirent(&sde, osDirname, childName) 71 | if err != nil { 72 | _ = dh.Close() 73 | return nil, err 74 | } 75 | entries = append(entries, &Dirent{name: childName, path: osDirname, modeType: mt}) 76 | } 77 | } 78 | 79 | func readDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { 80 | var entries []string 81 | var workBuffer []byte 82 | var sde *syscall.Dirent 83 | 84 | dh, err := os.Open(osDirname) 85 | if err != nil { 86 | return nil, err 87 | } 88 | fd := int(dh.Fd()) 89 | 90 | if len(scratchBuffer) < MinimumScratchBufferSize { 91 | scratchBuffer = newScratchBuffer() 92 | } 93 | 94 | for { 95 | if len(workBuffer) == 0 { 96 | n, err := syscall.ReadDirent(fd, scratchBuffer) 97 | // n, err := unix.ReadDirent(fd, scratchBuffer) 98 | if err != nil { 99 | if err == syscall.EINTR /* || err == unix.EINTR */ { 100 | continue 101 | } 102 | _ = dh.Close() 103 | return nil, err 104 | } 105 | if n <= 0 { // end of directory: normal exit 106 | if err = dh.Close(); err != nil { 107 | return nil, err 108 | } 109 | return entries, nil 110 | } 111 | workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read 112 | } 113 | 114 | sde = (*syscall.Dirent)(unsafe.Pointer(&workBuffer[0])) // point entry to first syscall.Dirent in buffer 115 | // Handle first entry in the work buffer. 116 | workBuffer = workBuffer[reclen(sde):] // advance buffer for next iteration through loop 117 | 118 | if inoFromDirent(sde) == 0 { 119 | continue // inode set to 0 indicates an entry that was marked as deleted 120 | } 121 | 122 | nameSlice := nameFromDirent(sde) 123 | nameLength := len(nameSlice) 124 | 125 | if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { 126 | continue 127 | } 128 | 129 | entries = append(entries, string(nameSlice)) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /readdir_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package godirwalk 4 | 5 | import "os" 6 | 7 | // MinimumScratchBufferSize specifies the minimum size of the scratch buffer 8 | // that ReadDirents, ReadDirnames, Scanner, and Walk will use when reading file 9 | // entries from the operating system. During program startup it is initialized 10 | // to the result from calling `os.Getpagesize()` for non Windows environments, 11 | // and 0 for Windows. 12 | var MinimumScratchBufferSize = 0 13 | 14 | func newScratchBuffer() []byte { return nil } 15 | 16 | func readDirents(osDirname string, _ []byte) ([]*Dirent, error) { 17 | dh, err := os.Open(osDirname) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | fileinfos, err := dh.Readdir(-1) 23 | if err != nil { 24 | _ = dh.Close() 25 | return nil, err 26 | } 27 | 28 | entries := make([]*Dirent, len(fileinfos)) 29 | 30 | for i, fi := range fileinfos { 31 | entries[i] = &Dirent{ 32 | name: fi.Name(), 33 | path: osDirname, 34 | modeType: fi.Mode() & os.ModeType, 35 | } 36 | } 37 | 38 | if err = dh.Close(); err != nil { 39 | return nil, err 40 | } 41 | return entries, nil 42 | } 43 | 44 | func readDirnames(osDirname string, _ []byte) ([]string, error) { 45 | dh, err := os.Open(osDirname) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | fileinfos, err := dh.Readdir(-1) 51 | if err != nil { 52 | _ = dh.Close() 53 | return nil, err 54 | } 55 | 56 | entries := make([]string, len(fileinfos)) 57 | 58 | for i, fi := range fileinfos { 59 | entries[i] = fi.Name() 60 | } 61 | 62 | if err = dh.Close(); err != nil { 63 | return nil, err 64 | } 65 | return entries, nil 66 | } 67 | -------------------------------------------------------------------------------- /reclenFromNamlen.go: -------------------------------------------------------------------------------- 1 | // +build dragonfly 2 | 3 | package godirwalk 4 | 5 | import "syscall" 6 | 7 | func reclen(de *syscall.Dirent) uint64 { 8 | return (16 + uint64(de.Namlen) + 1 + 7) &^ 7 9 | } 10 | -------------------------------------------------------------------------------- /reclenFromReclen.go: -------------------------------------------------------------------------------- 1 | // +build nacl linux js solaris aix darwin freebsd netbsd openbsd 2 | 3 | package godirwalk 4 | 5 | import "syscall" 6 | 7 | func reclen(de *syscall.Dirent) uint64 { 8 | return uint64(de.Reclen) 9 | } 10 | -------------------------------------------------------------------------------- /scaffoling_test.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | ) 11 | 12 | // maxName is the tested maximum length of a filename this library will 13 | // handle. Previous attempts to set it to one less than the size of 14 | // syscall.Dirent.Name array resulted in runtime errors trying to create 15 | // a test scaffolding file whose size exceeded 255 bytes. This filename 16 | // is 255 characters long. 17 | const maxName = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" 18 | 19 | // scaffolingRoot is the temporary directory root for scaffold directory. 20 | var scaffolingRoot string 21 | 22 | func TestMain(m *testing.M) { 23 | flag.Parse() 24 | 25 | var code int // program exit code 26 | 27 | // All tests use the same directory test scaffolding. Create the directory 28 | // hierarchy, run the tests, then remove the root directory of the test 29 | // scaffolding. 30 | 31 | defer func() { 32 | if err := teardown(); err != nil { 33 | fmt.Fprintf(os.Stderr, "godirwalk teardown: %s\n", err) 34 | code = 1 35 | } 36 | os.Exit(code) 37 | }() 38 | 39 | // When cannot complete setup, dump the directory so we see what we have, 40 | // then bail. 41 | if err := setup(); err != nil { 42 | fmt.Fprintf(os.Stderr, "godirwalk setup: %s\n", err) 43 | dumpDirectory() 44 | code = 1 45 | return 46 | } 47 | 48 | code = m.Run() 49 | 50 | // When any test was a failure, then use standard library to walk test 51 | // scaffolding directory and print its contents. 52 | if code != 0 { 53 | dumpDirectory() 54 | } 55 | } 56 | 57 | func setup() error { 58 | var err error 59 | 60 | scaffolingRoot, err = ioutil.TempDir(os.TempDir(), "godirwalk-") 61 | if err != nil { 62 | return err 63 | } 64 | 65 | entries := []Creater{ 66 | file{"d0/" + maxName}, 67 | file{"d0/f0"}, // will be deleted after symlink for it created 68 | file{"d0/f1"}, // 69 | file{"d0/d1/f2"}, // 70 | file{"d0/skips/d2/f3"}, // node precedes skip 71 | file{"d0/skips/d2/skip"}, // skip is non-directory 72 | file{"d0/skips/d2/z1"}, // node follows skip non-directory: should never be visited 73 | file{"d0/skips/d3/f4"}, // node precedes skip 74 | file{"d0/skips/d3/skip/f5"}, // skip is directory: this node should never be visited 75 | file{"d0/skips/d3/z2"}, // node follows skip directory: should be visited 76 | 77 | link{"d0/symlinks/nothing", "../f0"}, // referent will be deleted 78 | link{"d0/symlinks/toF1", "../f1"}, // 79 | link{"d0/symlinks/toD1", "../d1"}, // 80 | link{"d0/symlinks/d4/toSD1", "../toD1"}, // chained symbolic links 81 | link{"d0/symlinks/d4/toSF1", "../toF1"}, // chained symbolic links 82 | } 83 | 84 | for _, entry := range entries { 85 | if err := entry.Create(); err != nil { 86 | return fmt.Errorf("cannot create scaffolding entry: %s", err) 87 | } 88 | } 89 | 90 | oldname, err := filepath.Abs(filepath.Join(scaffolingRoot, "d0/f1")) 91 | if err != nil { 92 | return fmt.Errorf("cannot create scaffolding entry: %s", err) 93 | } 94 | if err := (link{"d0/symlinks/toAbs", oldname}).Create(); err != nil { 95 | return fmt.Errorf("cannot create scaffolding entry: %s", err) 96 | } 97 | 98 | if err := os.Remove(filepath.Join(scaffolingRoot, "d0/f0")); err != nil { 99 | return fmt.Errorf("cannot remove file from test scaffolding: %s", err) 100 | } 101 | 102 | return nil 103 | } 104 | 105 | func teardown() error { 106 | if scaffolingRoot == "" { 107 | return nil // if we do not even have a test root directory then exit 108 | } 109 | if err := os.RemoveAll(scaffolingRoot); err != nil { 110 | return err 111 | } 112 | return nil 113 | } 114 | 115 | func dumpDirectory() { 116 | trim := len(scaffolingRoot) // trim rootDir from prefix of strings 117 | err := filepath.Walk(scaffolingRoot, func(osPathname string, info os.FileInfo, err error) error { 118 | if err != nil { 119 | // we have no info, so get it 120 | info, err2 := os.Lstat(osPathname) 121 | if err2 != nil { 122 | fmt.Fprintf(os.Stderr, "?--------- %s: %s\n", osPathname[trim:], err2) 123 | } else { 124 | fmt.Fprintf(os.Stderr, "%s %s: %s\n", info.Mode(), osPathname[trim:], err) 125 | } 126 | return nil 127 | } 128 | 129 | var suffix string 130 | 131 | if info.Mode()&os.ModeSymlink != 0 { 132 | referent, err := os.Readlink(osPathname) 133 | if err != nil { 134 | suffix = fmt.Sprintf(": cannot read symlink: %s", err) 135 | err = nil 136 | } else { 137 | suffix = fmt.Sprintf(" -> %s", referent) 138 | } 139 | } 140 | fmt.Fprintf(os.Stderr, "%s %s%s\n", info.Mode(), osPathname[trim:], suffix) 141 | return nil 142 | }) 143 | if err != nil { 144 | fmt.Fprintf(os.Stderr, "cannot walk test directory: %s\n", err) 145 | } 146 | } 147 | 148 | //////////////////////////////////////// 149 | // helpers to create file system entries for test scaffolding 150 | 151 | type Creater interface { 152 | Create() error 153 | } 154 | 155 | type file struct { 156 | name string 157 | } 158 | 159 | func (f file) Create() error { 160 | newname := filepath.Join(scaffolingRoot, filepath.FromSlash(f.name)) 161 | if err := os.MkdirAll(filepath.Dir(newname), os.ModePerm); err != nil { 162 | return fmt.Errorf("cannot create directory for test scaffolding: %s", err) 163 | } 164 | if err := ioutil.WriteFile(newname, []byte(newname+"\n"), os.ModePerm); err != nil { 165 | return fmt.Errorf("cannot create file for test scaffolding: %s", err) 166 | } 167 | return nil 168 | } 169 | 170 | type link struct { 171 | name, referent string 172 | } 173 | 174 | func (s link) Create() error { 175 | newname := filepath.Join(scaffolingRoot, filepath.FromSlash(s.name)) 176 | if err := os.MkdirAll(filepath.Dir(newname), os.ModePerm); err != nil { 177 | return fmt.Errorf("cannot create directory for test scaffolding: %s", err) 178 | } 179 | oldname := filepath.FromSlash(s.referent) 180 | if err := os.Symlink(oldname, newname); err != nil { 181 | return fmt.Errorf("cannot create symbolic link for test scaffolding: %s", err) 182 | } 183 | return nil 184 | } 185 | -------------------------------------------------------------------------------- /scandir_test.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestScanner(t *testing.T) { 10 | t.Run("collect names", func(t *testing.T) { 11 | var actual []string 12 | 13 | scanner, err := NewScanner(filepath.Join(scaffolingRoot, "d0")) 14 | ensureError(t, err) 15 | 16 | for scanner.Scan() { 17 | actual = append(actual, scanner.Name()) 18 | } 19 | ensureError(t, scanner.Err()) 20 | 21 | expected := []string{maxName, "d1", "f1", "skips", "symlinks"} 22 | ensureStringSlicesMatch(t, actual, expected) 23 | }) 24 | 25 | t.Run("collect dirents", func(t *testing.T) { 26 | var actual []*Dirent 27 | 28 | testroot := filepath.Join(scaffolingRoot, "d0") 29 | 30 | scanner, err := NewScanner(testroot) 31 | ensureError(t, err) 32 | 33 | for scanner.Scan() { 34 | dirent, err := scanner.Dirent() 35 | ensureError(t, err) 36 | actual = append(actual, dirent) 37 | } 38 | ensureError(t, scanner.Err()) 39 | 40 | expected := Dirents{ 41 | &Dirent{ 42 | name: maxName, 43 | path: testroot, 44 | modeType: os.FileMode(0), 45 | }, 46 | &Dirent{ 47 | name: "d1", 48 | path: testroot, 49 | modeType: os.ModeDir, 50 | }, 51 | &Dirent{ 52 | name: "f1", 53 | path: testroot, 54 | modeType: os.FileMode(0), 55 | }, 56 | &Dirent{ 57 | name: "skips", 58 | path: testroot, 59 | modeType: os.ModeDir, 60 | }, 61 | &Dirent{ 62 | name: "symlinks", 63 | path: testroot, 64 | modeType: os.ModeDir, 65 | }, 66 | } 67 | 68 | ensureDirentsMatch(t, actual, expected) 69 | }) 70 | 71 | t.Run("symlink to directory", func(t *testing.T) { 72 | scanner, err := NewScanner(filepath.Join(scaffolingRoot, "d0/symlinks")) 73 | ensureError(t, err) 74 | 75 | var found bool 76 | 77 | for scanner.Scan() { 78 | if scanner.Name() != "toD1" { 79 | continue 80 | } 81 | found = true 82 | 83 | de, err := scanner.Dirent() 84 | ensureError(t, err) 85 | 86 | got, err := de.IsDirOrSymlinkToDir() 87 | ensureError(t, err) 88 | 89 | if want := true; got != want { 90 | t.Errorf("GOT: %v; WANT: %v", got, want) 91 | } 92 | } 93 | 94 | ensureError(t, scanner.Err()) 95 | 96 | if got, want := found, true; got != want { 97 | t.Errorf("GOT: %v; WANT: %v", got, want) 98 | } 99 | }) 100 | } 101 | -------------------------------------------------------------------------------- /scandir_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package godirwalk 5 | 6 | import ( 7 | "os" 8 | "syscall" 9 | "unsafe" 10 | ) 11 | 12 | // Scanner is an iterator to enumerate the contents of a directory. 13 | type Scanner struct { 14 | scratchBuffer []byte // read directory bytes from file system into this buffer 15 | workBuffer []byte // points into scratchBuffer, from which we chunk out directory entries 16 | osDirname string 17 | childName string 18 | err error // err is the error associated with scanning directory 19 | statErr error // statErr is any error return while attempting to stat an entry 20 | dh *os.File // used to close directory after done reading 21 | de *Dirent // most recently decoded directory entry 22 | sde syscall.Dirent 23 | fd int // file descriptor used to read entries from directory 24 | } 25 | 26 | // NewScanner returns a new directory Scanner that lazily enumerates 27 | // the contents of a single directory. To prevent resource leaks, 28 | // caller must invoke either the Scanner's Close or Err method after 29 | // it has completed scanning a directory. 30 | // 31 | // scanner, err := godirwalk.NewScanner(dirname) 32 | // if err != nil { 33 | // fatal("cannot scan directory: %s", err) 34 | // } 35 | // 36 | // for scanner.Scan() { 37 | // dirent, err := scanner.Dirent() 38 | // if err != nil { 39 | // warning("cannot get dirent: %s", err) 40 | // continue 41 | // } 42 | // name := dirent.Name() 43 | // if name == "break" { 44 | // break 45 | // } 46 | // if name == "continue" { 47 | // continue 48 | // } 49 | // fmt.Printf("%v %v\n", dirent.ModeType(), dirent.Name()) 50 | // } 51 | // if err := scanner.Err(); err != nil { 52 | // fatal("cannot scan directory: %s", err) 53 | // } 54 | func NewScanner(osDirname string) (*Scanner, error) { 55 | return NewScannerWithScratchBuffer(osDirname, nil) 56 | } 57 | 58 | // NewScannerWithScratchBuffer returns a new directory Scanner that 59 | // lazily enumerates the contents of a single directory. On platforms 60 | // other than Windows it uses the provided scratch buffer to read from 61 | // the file system. On Windows the scratch buffer is ignored. To 62 | // prevent resource leaks, caller must invoke either the Scanner's 63 | // Close or Err method after it has completed scanning a directory. 64 | func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) { 65 | dh, err := os.Open(osDirname) 66 | if err != nil { 67 | return nil, err 68 | } 69 | if len(scratchBuffer) < MinimumScratchBufferSize { 70 | scratchBuffer = newScratchBuffer() 71 | } 72 | scanner := &Scanner{ 73 | scratchBuffer: scratchBuffer, 74 | osDirname: osDirname, 75 | dh: dh, 76 | fd: int(dh.Fd()), 77 | } 78 | return scanner, nil 79 | } 80 | 81 | // Close releases resources associated with scanning a directory. Call 82 | // either this or the Err method when the directory no longer needs to 83 | // be scanned. 84 | func (s *Scanner) Close() error { 85 | return s.Err() 86 | } 87 | 88 | // Dirent returns the current directory entry while scanning a directory. 89 | func (s *Scanner) Dirent() (*Dirent, error) { 90 | if s.de == nil { 91 | s.de = &Dirent{name: s.childName, path: s.osDirname} 92 | s.de.modeType, s.statErr = modeTypeFromDirent(&s.sde, s.osDirname, s.childName) 93 | } 94 | return s.de, s.statErr 95 | } 96 | 97 | // done is called when directory scanner unable to continue, with either the 98 | // triggering error, or nil when there are simply no more entries to read from 99 | // the directory. 100 | func (s *Scanner) done(err error) { 101 | if s.dh == nil { 102 | return 103 | } 104 | 105 | s.err = err 106 | 107 | if err = s.dh.Close(); s.err == nil { 108 | s.err = err 109 | } 110 | 111 | s.osDirname, s.childName = "", "" 112 | s.scratchBuffer, s.workBuffer = nil, nil 113 | s.dh, s.de, s.statErr = nil, nil, nil 114 | s.sde = syscall.Dirent{} 115 | s.fd = 0 116 | } 117 | 118 | // Err returns any error associated with scanning a directory. It is 119 | // normal to call Err after Scan returns false, even though they both 120 | // ensure Scanner resources are released. Call either this or the 121 | // Close method when the directory no longer needs to be scanned. 122 | func (s *Scanner) Err() error { 123 | s.done(nil) 124 | return s.err 125 | } 126 | 127 | // Name returns the base name of the current directory entry while scanning a 128 | // directory. 129 | func (s *Scanner) Name() string { return s.childName } 130 | 131 | // Scan potentially reads and then decodes the next directory entry from the 132 | // file system. 133 | // 134 | // When it returns false, this releases resources used by the Scanner then 135 | // returns any error associated with closing the file system directory resource. 136 | func (s *Scanner) Scan() bool { 137 | if s.dh == nil { 138 | return false 139 | } 140 | 141 | s.de = nil 142 | 143 | for { 144 | // When the work buffer has nothing remaining to decode, we need to load 145 | // more data from disk. 146 | if len(s.workBuffer) == 0 { 147 | n, err := syscall.ReadDirent(s.fd, s.scratchBuffer) 148 | // n, err := unix.ReadDirent(s.fd, s.scratchBuffer) 149 | if err != nil { 150 | if err == syscall.EINTR /* || err == unix.EINTR */ { 151 | continue 152 | } 153 | s.done(err) // any other error forces a stop 154 | return false 155 | } 156 | if n <= 0 { // end of directory: normal exit 157 | s.done(nil) 158 | return false 159 | } 160 | s.workBuffer = s.scratchBuffer[:n] // trim work buffer to number of bytes read 161 | } 162 | 163 | // point entry to first syscall.Dirent in buffer 164 | copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(&s.sde))[:], s.workBuffer) 165 | s.workBuffer = s.workBuffer[reclen(&s.sde):] // advance buffer for next iteration through loop 166 | 167 | if inoFromDirent(&s.sde) == 0 { 168 | continue // inode set to 0 indicates an entry that was marked as deleted 169 | } 170 | 171 | nameSlice := nameFromDirent(&s.sde) 172 | nameLength := len(nameSlice) 173 | 174 | if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { 175 | continue 176 | } 177 | 178 | s.childName = string(nameSlice) 179 | return true 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /scandir_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package godirwalk 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | ) 10 | 11 | // Scanner is an iterator to enumerate the contents of a directory. 12 | type Scanner struct { 13 | osDirname string 14 | childName string 15 | dh *os.File // dh is handle to open directory 16 | de *Dirent 17 | err error // err is the error associated with scanning directory 18 | childMode os.FileMode 19 | } 20 | 21 | // NewScanner returns a new directory Scanner that lazily enumerates 22 | // the contents of a single directory. To prevent resource leaks, 23 | // caller must invoke either the Scanner's Close or Err method after 24 | // it has completed scanning a directory. 25 | // 26 | // scanner, err := godirwalk.NewScanner(dirname) 27 | // if err != nil { 28 | // fatal("cannot scan directory: %s", err) 29 | // } 30 | // 31 | // for scanner.Scan() { 32 | // dirent, err := scanner.Dirent() 33 | // if err != nil { 34 | // warning("cannot get dirent: %s", err) 35 | // continue 36 | // } 37 | // name := dirent.Name() 38 | // if name == "break" { 39 | // break 40 | // } 41 | // if name == "continue" { 42 | // continue 43 | // } 44 | // fmt.Printf("%v %v\n", dirent.ModeType(), dirent.Name()) 45 | // } 46 | // if err := scanner.Err(); err != nil { 47 | // fatal("cannot scan directory: %s", err) 48 | // } 49 | func NewScanner(osDirname string) (*Scanner, error) { 50 | dh, err := os.Open(osDirname) 51 | if err != nil { 52 | return nil, err 53 | } 54 | scanner := &Scanner{ 55 | osDirname: osDirname, 56 | dh: dh, 57 | } 58 | return scanner, nil 59 | } 60 | 61 | // NewScannerWithScratchBuffer returns a new directory Scanner that 62 | // lazily enumerates the contents of a single directory. On platforms 63 | // other than Windows it uses the provided scratch buffer to read from 64 | // the file system. On Windows the scratch buffer parameter is 65 | // ignored. To prevent resource leaks, caller must invoke either the 66 | // Scanner's Close or Err method after it has completed scanning a 67 | // directory. 68 | func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) { 69 | return NewScanner(osDirname) 70 | } 71 | 72 | // Close releases resources associated with scanning a directory. Call 73 | // either this or the Err method when the directory no longer needs to 74 | // be scanned. 75 | func (s *Scanner) Close() error { 76 | return s.Err() 77 | } 78 | 79 | // Dirent returns the current directory entry while scanning a directory. 80 | func (s *Scanner) Dirent() (*Dirent, error) { 81 | if s.de == nil { 82 | s.de = &Dirent{ 83 | name: s.childName, 84 | path: s.osDirname, 85 | modeType: s.childMode, 86 | } 87 | } 88 | return s.de, nil 89 | } 90 | 91 | // done is called when directory scanner unable to continue, with either the 92 | // triggering error, or nil when there are simply no more entries to read from 93 | // the directory. 94 | func (s *Scanner) done(err error) { 95 | if s.dh == nil { 96 | return 97 | } 98 | 99 | s.err = err 100 | 101 | if err = s.dh.Close(); s.err == nil { 102 | s.err = err 103 | } 104 | 105 | s.childName, s.osDirname = "", "" 106 | s.de, s.dh = nil, nil 107 | } 108 | 109 | // Err returns any error associated with scanning a directory. It is 110 | // normal to call Err after Scan returns false, even though they both 111 | // ensure Scanner resources are released. Call either this or the 112 | // Close method when the directory no longer needs to be scanned. 113 | func (s *Scanner) Err() error { 114 | s.done(nil) 115 | return s.err 116 | } 117 | 118 | // Name returns the base name of the current directory entry while scanning a 119 | // directory. 120 | func (s *Scanner) Name() string { return s.childName } 121 | 122 | // Scan potentially reads and then decodes the next directory entry from the 123 | // file system. 124 | // 125 | // When it returns false, this releases resources used by the Scanner then 126 | // returns any error associated with closing the file system directory resource. 127 | func (s *Scanner) Scan() bool { 128 | if s.dh == nil { 129 | return false 130 | } 131 | 132 | s.de = nil 133 | 134 | fileinfos, err := s.dh.Readdir(1) 135 | if err != nil { 136 | s.done(err) 137 | return false 138 | } 139 | 140 | if l := len(fileinfos); l != 1 { 141 | s.done(fmt.Errorf("expected a single entry rather than %d", l)) 142 | return false 143 | } 144 | 145 | fi := fileinfos[0] 146 | s.childMode = fi.Mode() & os.ModeType 147 | s.childName = fi.Name() 148 | return true 149 | } 150 | -------------------------------------------------------------------------------- /scanner.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import "sort" 4 | 5 | type scanner interface { 6 | Dirent() (*Dirent, error) 7 | Err() error 8 | Name() string 9 | Scan() bool 10 | } 11 | 12 | // sortedScanner enumerates through a directory's contents after reading the 13 | // entire directory and sorting the entries by name. Used by walk to simplify 14 | // its implementation. 15 | type sortedScanner struct { 16 | dd []*Dirent 17 | de *Dirent 18 | } 19 | 20 | func newSortedScanner(osPathname string, scratchBuffer []byte) (*sortedScanner, error) { 21 | deChildren, err := ReadDirents(osPathname, scratchBuffer) 22 | if err != nil { 23 | return nil, err 24 | } 25 | sort.Sort(deChildren) 26 | return &sortedScanner{dd: deChildren}, nil 27 | } 28 | 29 | func (d *sortedScanner) Err() error { 30 | d.dd, d.de = nil, nil 31 | return nil 32 | } 33 | 34 | func (d *sortedScanner) Dirent() (*Dirent, error) { return d.de, nil } 35 | 36 | func (d *sortedScanner) Name() string { return d.de.name } 37 | 38 | func (d *sortedScanner) Scan() bool { 39 | if len(d.dd) > 0 { 40 | d.de, d.dd = d.dd[0], d.dd[1:] 41 | return true 42 | } 43 | return false 44 | } 45 | -------------------------------------------------------------------------------- /walk.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | ) 9 | 10 | // Options provide parameters for how the Walk function operates. 11 | type Options struct { 12 | // ErrorCallback specifies a function to be invoked in the case of an error 13 | // that could potentially be ignored while walking a file system 14 | // hierarchy. When set to nil or left as its zero-value, any error condition 15 | // causes Walk to immediately return the error describing what took 16 | // place. When non-nil, this user supplied function is invoked with the OS 17 | // pathname of the file system object that caused the error along with the 18 | // error that took place. The return value of the supplied ErrorCallback 19 | // function determines whether the error will cause Walk to halt immediately 20 | // as it would were no ErrorCallback value provided, or skip this file 21 | // system node yet continue on with the remaining nodes in the file system 22 | // hierarchy. 23 | // 24 | // ErrorCallback is invoked both for errors that are returned by the 25 | // runtime, and for errors returned by other user supplied callback 26 | // functions. 27 | ErrorCallback func(string, error) ErrorAction 28 | 29 | // FollowSymbolicLinks specifies whether Walk will follow symbolic links 30 | // that refer to directories. When set to false or left as its zero-value, 31 | // Walk will still invoke the callback function with symbolic link nodes, 32 | // but if the symbolic link refers to a directory, it will not recurse on 33 | // that directory. When set to true, Walk will recurse on symbolic links 34 | // that refer to a directory. 35 | FollowSymbolicLinks bool 36 | 37 | // Unsorted controls whether or not Walk will sort the immediate descendants 38 | // of a directory by their relative names prior to visiting each of those 39 | // entries. 40 | // 41 | // When set to false or left at its zero-value, Walk will get the list of 42 | // immediate descendants of a particular directory, sort that list by 43 | // lexical order of their names, and then visit each node in the list in 44 | // sorted order. This will cause Walk to always traverse the same directory 45 | // tree in the same order, however may be inefficient for directories with 46 | // many immediate descendants. 47 | // 48 | // When set to true, Walk skips sorting the list of immediate descendants 49 | // for a directory, and simply visits each node in the order the operating 50 | // system enumerated them. This will be more fast, but with the side effect 51 | // that the traversal order may be different from one invocation to the 52 | // next. 53 | Unsorted bool 54 | 55 | // Callback is a required function that Walk will invoke for every file 56 | // system node it encounters. 57 | Callback WalkFunc 58 | 59 | // PostChildrenCallback is an option function that Walk will invoke for 60 | // every file system directory it encounters after its children have been 61 | // processed. 62 | PostChildrenCallback WalkFunc 63 | 64 | // ScratchBuffer is an optional byte slice to use as a scratch buffer for 65 | // Walk to use when reading directory entries, to reduce amount of garbage 66 | // generation. Not all architectures take advantage of the scratch 67 | // buffer. If omitted or the provided buffer has fewer bytes than 68 | // MinimumScratchBufferSize, then a buffer with MinimumScratchBufferSize 69 | // bytes will be created and used once per Walk invocation. 70 | ScratchBuffer []byte 71 | 72 | // AllowNonDirectory causes Walk to bypass the check that ensures it is 73 | // being called on a directory node, or when FollowSymbolicLinks is true, a 74 | // symbolic link that points to a directory. Leave this value false to have 75 | // Walk return an error when called on a non-directory. Set this true to 76 | // have Walk run even when called on a non-directory node. 77 | AllowNonDirectory bool 78 | } 79 | 80 | // ErrorAction defines a set of actions the Walk function could take based on 81 | // the occurrence of an error while walking the file system. See the 82 | // documentation for the ErrorCallback field of the Options structure for more 83 | // information. 84 | type ErrorAction int 85 | 86 | const ( 87 | // Halt is the ErrorAction return value when the upstream code wants to halt 88 | // the walk process when a runtime error takes place. It matches the default 89 | // action the Walk function would take were no ErrorCallback provided. 90 | Halt ErrorAction = iota 91 | 92 | // SkipNode is the ErrorAction return value when the upstream code wants to 93 | // ignore the runtime error for the current file system node, skip 94 | // processing of the node that caused the error, and continue walking the 95 | // file system hierarchy with the remaining nodes. 96 | SkipNode 97 | ) 98 | 99 | // SkipThis is used as a return value from WalkFuncs to indicate that the file 100 | // system entry named in the call is to be skipped. It is not returned as an 101 | // error by any function. 102 | var SkipThis = errors.New("skip this directory entry") 103 | 104 | // WalkFunc is the type of the function called for each file system node visited 105 | // by Walk. The pathname argument will contain the argument to Walk as a prefix; 106 | // that is, if Walk is called with "dir", which is a directory containing the 107 | // file "a", the provided WalkFunc will be invoked with the argument "dir/a", 108 | // using the correct os.PathSeparator for the Go Operating System architecture, 109 | // GOOS. The directory entry argument is a pointer to a Dirent for the node, 110 | // providing access to both the basename and the mode type of the file system 111 | // node. 112 | // 113 | // If an error is returned by the Callback or PostChildrenCallback functions, 114 | // and no ErrorCallback function is provided, processing stops. If an 115 | // ErrorCallback function is provided, then it is invoked with the OS pathname 116 | // of the node that caused the error along along with the error. The return 117 | // value of the ErrorCallback function determines whether to halt processing, or 118 | // skip this node and continue processing remaining file system nodes. 119 | // 120 | // The exception is when the function returns the special value 121 | // filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a 122 | // directory, Walk skips the directory's contents entirely. If the function 123 | // returns filepath.SkipDir when invoked on a non-directory file system node, 124 | // Walk skips the remaining files in the containing directory. Note that any 125 | // supplied ErrorCallback function is not invoked with filepath.SkipDir when the 126 | // Callback or PostChildrenCallback functions return that special value. 127 | // 128 | // One arguably confusing aspect of the filepath.WalkFunc API that this library 129 | // must emulate is how a caller tells Walk to skip file system entries or 130 | // directories. With both filepath.Walk and this Walk, when a callback function 131 | // wants to skip a directory and not descend into its children, it returns 132 | // filepath.SkipDir. If the callback function returns filepath.SkipDir for a 133 | // non-directory, filepath.Walk and this library will stop processing any more 134 | // entries in the current directory, which is what many people do not want. If 135 | // you want to simply skip a particular non-directory entry but continue 136 | // processing entries in the directory, a callback function must return nil. The 137 | // implications of this API is when you want to walk a file system hierarchy and 138 | // skip an entry, when the entry is a directory, you must return one value, 139 | // namely filepath.SkipDir, but when the entry is a non-directory, you must 140 | // return a different value, namely nil. In other words, to get identical 141 | // behavior for two file system entry types you need to send different token 142 | // values. 143 | // 144 | // Here is an example callback function that adheres to filepath.Walk API to 145 | // have it skip any file system entry whose full pathname includes a particular 146 | // substring, optSkip: 147 | // 148 | // func callback1(osPathname string, de *godirwalk.Dirent) error { 149 | // if optSkip != "" && strings.Contains(osPathname, optSkip) { 150 | // if b, err := de.IsDirOrSymlinkToDir(); b == true && err == nil { 151 | // return filepath.SkipDir 152 | // } 153 | // return nil 154 | // } 155 | // // Process file like normal... 156 | // return nil 157 | // } 158 | // 159 | // This library attempts to eliminate some of that logic boilerplate by 160 | // providing a new token error value, SkipThis, which a callback function may 161 | // return to skip the current file system entry regardless of what type of entry 162 | // it is. If the current entry is a directory, its children will not be 163 | // enumerated, exactly as if the callback returned filepath.SkipDir. If the 164 | // current entry is a non-directory, the next file system entry in the current 165 | // directory will be enumerated, exactly as if the callback returned nil. The 166 | // following example callback function has identical behavior as the previous, 167 | // but has less boilerplate, and admittedly more simple logic. 168 | // 169 | // func callback2(osPathname string, de *godirwalk.Dirent) error { 170 | // if optSkip != "" && strings.Contains(osPathname, optSkip) { 171 | // return godirwalk.SkipThis 172 | // } 173 | // // Process file like normal... 174 | // return nil 175 | // } 176 | type WalkFunc func(osPathname string, directoryEntry *Dirent) error 177 | 178 | // Walk walks the file tree rooted at the specified directory, calling the 179 | // specified callback function for each file system node in the tree, including 180 | // root, symbolic links, and other node types. 181 | // 182 | // This function is often much faster than filepath.Walk because it does not 183 | // invoke os.Stat for every node it encounters, but rather obtains the file 184 | // system node type when it reads the parent directory. 185 | // 186 | // If a runtime error occurs, either from the operating system or from the 187 | // upstream Callback or PostChildrenCallback functions, processing typically 188 | // halts. However, when an ErrorCallback function is provided in the provided 189 | // Options structure, that function is invoked with the error along with the OS 190 | // pathname of the file system node that caused the error. The ErrorCallback 191 | // function's return value determines the action that Walk will then take. 192 | // 193 | // func main() { 194 | // dirname := "." 195 | // if len(os.Args) > 1 { 196 | // dirname = os.Args[1] 197 | // } 198 | // err := godirwalk.Walk(dirname, &godirwalk.Options{ 199 | // Callback: func(osPathname string, de *godirwalk.Dirent) error { 200 | // fmt.Printf("%s %s\n", de.ModeType(), osPathname) 201 | // return nil 202 | // }, 203 | // ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { 204 | // // Your program may want to log the error somehow. 205 | // fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) 206 | // 207 | // // For the purposes of this example, a simple SkipNode will suffice, 208 | // // although in reality perhaps additional logic might be called for. 209 | // return godirwalk.SkipNode 210 | // }, 211 | // }) 212 | // if err != nil { 213 | // fmt.Fprintf(os.Stderr, "%s\n", err) 214 | // os.Exit(1) 215 | // } 216 | // } 217 | func Walk(pathname string, options *Options) error { 218 | if options == nil || options.Callback == nil { 219 | return errors.New("cannot walk without non-nil options and Callback function") 220 | } 221 | 222 | pathname = filepath.Clean(pathname) 223 | 224 | var fi os.FileInfo 225 | var err error 226 | 227 | if options.FollowSymbolicLinks { 228 | fi, err = os.Stat(pathname) 229 | } else { 230 | fi, err = os.Lstat(pathname) 231 | } 232 | if err != nil { 233 | return err 234 | } 235 | 236 | mode := fi.Mode() 237 | if !options.AllowNonDirectory && mode&os.ModeDir == 0 { 238 | return fmt.Errorf("cannot Walk non-directory: %s", pathname) 239 | } 240 | 241 | dirent := &Dirent{ 242 | name: filepath.Base(pathname), 243 | path: filepath.Dir(pathname), 244 | modeType: mode & os.ModeType, 245 | } 246 | 247 | if len(options.ScratchBuffer) < MinimumScratchBufferSize { 248 | options.ScratchBuffer = newScratchBuffer() 249 | } 250 | 251 | // If ErrorCallback is nil, set to a default value that halts the walk 252 | // process on all operating system errors. This is done to allow error 253 | // handling to be more succinct in the walk code. 254 | if options.ErrorCallback == nil { 255 | options.ErrorCallback = defaultErrorCallback 256 | } 257 | 258 | err = walk(pathname, dirent, options) 259 | switch err { 260 | case nil, SkipThis, filepath.SkipDir: 261 | // silence SkipThis and filepath.SkipDir for top level 262 | debug("no error of significance: %v\n", err) 263 | return nil 264 | default: 265 | return err 266 | } 267 | } 268 | 269 | // defaultErrorCallback always returns Halt because if the upstream code did not 270 | // provide an ErrorCallback function, walking the file system hierarchy ought to 271 | // halt upon any operating system error. 272 | func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt } 273 | 274 | // walk recursively traverses the file system node specified by pathname and the 275 | // Dirent. 276 | func walk(osPathname string, dirent *Dirent, options *Options) error { 277 | err := options.Callback(osPathname, dirent) 278 | if err != nil { 279 | if err == SkipThis || err == filepath.SkipDir { 280 | return err 281 | } 282 | if action := options.ErrorCallback(osPathname, err); action == SkipNode { 283 | return nil 284 | } 285 | return err 286 | } 287 | 288 | if dirent.IsSymlink() { 289 | if !options.FollowSymbolicLinks { 290 | return nil 291 | } 292 | // Does this symlink point to a directory? 293 | info, err := os.Stat(osPathname) 294 | if err != nil { 295 | if action := options.ErrorCallback(osPathname, err); action == SkipNode { 296 | return nil 297 | } 298 | return err 299 | } 300 | if !info.IsDir() { 301 | return nil 302 | } 303 | } else if !dirent.IsDir() { 304 | return nil 305 | } 306 | 307 | // If get here, then specified pathname refers to a directory or a 308 | // symbolic link to a directory. 309 | 310 | var ds scanner 311 | 312 | if options.Unsorted { 313 | // When upstream does not request a sorted iteration, it's more memory 314 | // efficient to read a single child at a time from the file system. 315 | ds, err = NewScanner(osPathname) 316 | } else { 317 | // When upstream wants a sorted iteration, we must read the entire 318 | // directory and sort through the child names, and then iterate on each 319 | // child. 320 | ds, err = newSortedScanner(osPathname, options.ScratchBuffer) 321 | } 322 | if err != nil { 323 | if action := options.ErrorCallback(osPathname, err); action == SkipNode { 324 | return nil 325 | } 326 | return err 327 | } 328 | 329 | for ds.Scan() { 330 | deChild, err := ds.Dirent() 331 | osChildname := filepath.Join(osPathname, deChild.name) 332 | if err != nil { 333 | if action := options.ErrorCallback(osChildname, err); action == SkipNode { 334 | return nil 335 | } 336 | return err 337 | } 338 | err = walk(osChildname, deChild, options) 339 | debug("osChildname: %q; error: %v\n", osChildname, err) 340 | if err == nil || err == SkipThis { 341 | continue 342 | } 343 | if err != filepath.SkipDir { 344 | return err 345 | } 346 | // When received SkipDir on a directory or a symbolic link to a 347 | // directory, stop processing that directory but continue processing 348 | // siblings. When received on a non-directory, stop processing 349 | // remaining siblings. 350 | isDir, err := deChild.IsDirOrSymlinkToDir() 351 | if err != nil { 352 | if action := options.ErrorCallback(osChildname, err); action == SkipNode { 353 | continue // ignore and continue with next sibling 354 | } 355 | return err // caller does not approve of this error 356 | } 357 | if !isDir { 358 | break // stop processing remaining siblings, but allow post children callback 359 | } 360 | // continue processing remaining siblings 361 | } 362 | if err = ds.Err(); err != nil { 363 | return err 364 | } 365 | 366 | if options.PostChildrenCallback == nil { 367 | return nil 368 | } 369 | 370 | err = options.PostChildrenCallback(osPathname, dirent) 371 | if err == nil || err == filepath.SkipDir { 372 | return err 373 | } 374 | 375 | if action := options.ErrorCallback(osPathname, err); action == SkipNode { 376 | return nil 377 | } 378 | return err 379 | } 380 | -------------------------------------------------------------------------------- /walk_test.go: -------------------------------------------------------------------------------- 1 | package godirwalk 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "sort" 7 | "testing" 8 | ) 9 | 10 | func filepathWalk(tb testing.TB, osDirname string) []string { 11 | tb.Helper() 12 | var entries []string 13 | err := filepath.Walk(osDirname, func(osPathname string, info os.FileInfo, err error) error { 14 | if err != nil { 15 | return err 16 | } 17 | if info.Name() == "skip" { 18 | return filepath.SkipDir 19 | } 20 | entries = append(entries, filepath.FromSlash(osPathname)) 21 | return nil 22 | }) 23 | ensureError(tb, err) 24 | return entries 25 | } 26 | 27 | func godirwalkWalk(tb testing.TB, osDirname string) []string { 28 | tb.Helper() 29 | var entries []string 30 | err := Walk(osDirname, &Options{ 31 | Callback: func(osPathname string, dirent *Dirent) error { 32 | if dirent.Name() == "skip" { 33 | return filepath.SkipDir 34 | } 35 | entries = append(entries, filepath.FromSlash(osPathname)) 36 | return nil 37 | }, 38 | }) 39 | ensureError(tb, err) 40 | return entries 41 | } 42 | 43 | func godirwalkWalkUnsorted(tb testing.TB, osDirname string) []string { 44 | tb.Helper() 45 | var entries []string 46 | err := Walk(osDirname, &Options{ 47 | Callback: func(osPathname string, dirent *Dirent) error { 48 | if dirent.Name() == "skip" { 49 | return filepath.SkipDir 50 | } 51 | entries = append(entries, filepath.FromSlash(osPathname)) 52 | return nil 53 | }, 54 | Unsorted: true, 55 | }) 56 | ensureError(tb, err) 57 | return entries 58 | } 59 | 60 | // Ensure the results from calling this library's Walk function exactly match 61 | // those returned by filepath.Walk 62 | func ensureSameAsStandardLibrary(tb testing.TB, osDirname string) { 63 | tb.Helper() 64 | osDirname = filepath.Join(scaffolingRoot, osDirname) 65 | actual := godirwalkWalk(tb, osDirname) 66 | sort.Strings(actual) 67 | expected := filepathWalk(tb, osDirname) 68 | ensureStringSlicesMatch(tb, actual, expected) 69 | } 70 | 71 | // Test the entire test root hierarchy with all of its artifacts. This library 72 | // advertises itself as visiting the same file system entries as the standard 73 | // library, and responding to discovered errors the same way, including 74 | // responding to filepath.SkipDir exactly like the standard library does. This 75 | // test ensures that behavior is correct by enumerating the contents of the test 76 | // root directory. 77 | func TestWalkCompatibleWithFilepathWalk(t *testing.T) { 78 | t.Run("test root", func(t *testing.T) { 79 | ensureSameAsStandardLibrary(t, "d0") 80 | }) 81 | t.Run("ignore skips", func(t *testing.T) { 82 | // When filepath.SkipDir is returned, the remainder of the children in a 83 | // directory are not visited. This causes results to be different when 84 | // visiting in lexicographical order or natural order. For this test, we 85 | // want to ensure godirwalk can optimize traversals when unsorted using 86 | // the Scanner, but recognize that we cannot test against standard 87 | // library when we skip any nodes within it. 88 | osDirname := filepath.Join(scaffolingRoot, "d0/d1") 89 | actual := godirwalkWalkUnsorted(t, osDirname) 90 | sort.Strings(actual) 91 | expected := filepathWalk(t, osDirname) 92 | ensureStringSlicesMatch(t, actual, expected) 93 | }) 94 | } 95 | 96 | // Test cases for encountering the filepath.SkipDir error at different 97 | // relative positions from the invocation argument. 98 | func TestWalkSkipDir(t *testing.T) { 99 | t.Run("skip file at root", func(t *testing.T) { 100 | ensureSameAsStandardLibrary(t, "d0/skips/d2") 101 | }) 102 | 103 | t.Run("skip dir at root", func(t *testing.T) { 104 | ensureSameAsStandardLibrary(t, "d0/skips/d3") 105 | }) 106 | 107 | t.Run("skip nodes under root", func(t *testing.T) { 108 | ensureSameAsStandardLibrary(t, "d0/skips") 109 | }) 110 | 111 | t.Run("SkipDirOnSymlink", func(t *testing.T) { 112 | var actual []string 113 | err := Walk(filepath.Join(scaffolingRoot, "d0/skips"), &Options{ 114 | Callback: func(osPathname string, dirent *Dirent) error { 115 | if dirent.Name() == "skip" { 116 | return filepath.SkipDir 117 | } 118 | actual = append(actual, filepath.FromSlash(osPathname)) 119 | return nil 120 | }, 121 | FollowSymbolicLinks: true, 122 | }) 123 | 124 | ensureError(t, err) 125 | 126 | expected := []string{ 127 | filepath.Join(scaffolingRoot, "d0/skips"), 128 | filepath.Join(scaffolingRoot, "d0/skips/d2"), 129 | filepath.Join(scaffolingRoot, "d0/skips/d2/f3"), 130 | filepath.Join(scaffolingRoot, "d0/skips/d3"), 131 | filepath.Join(scaffolingRoot, "d0/skips/d3/f4"), 132 | filepath.Join(scaffolingRoot, "d0/skips/d3/z2"), 133 | } 134 | 135 | ensureStringSlicesMatch(t, actual, expected) 136 | }) 137 | } 138 | 139 | func TestWalkSkipThis(t *testing.T) { 140 | t.Run("SkipThis", func(t *testing.T) { 141 | var actual []string 142 | err := Walk(filepath.Join(scaffolingRoot, "d0"), &Options{ 143 | Callback: func(osPathname string, dirent *Dirent) error { 144 | switch name := dirent.Name(); name { 145 | case "skips", "skip", "nothing": 146 | return SkipThis 147 | } 148 | actual = append(actual, filepath.FromSlash(osPathname)) 149 | return nil 150 | }, 151 | FollowSymbolicLinks: true, 152 | }) 153 | 154 | ensureError(t, err) 155 | 156 | expected := []string{ 157 | filepath.Join(scaffolingRoot, "d0"), 158 | filepath.Join(scaffolingRoot, "d0/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), 159 | filepath.Join(scaffolingRoot, "d0/f1"), 160 | filepath.Join(scaffolingRoot, "d0/d1"), 161 | filepath.Join(scaffolingRoot, "d0/d1/f2"), 162 | filepath.Join(scaffolingRoot, "d0/symlinks"), 163 | filepath.Join(scaffolingRoot, "d0/symlinks/d4"), 164 | filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSD1"), 165 | filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSD1/f2"), 166 | filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSF1"), 167 | filepath.Join(scaffolingRoot, "d0/symlinks/toAbs"), 168 | filepath.Join(scaffolingRoot, "d0/symlinks/toD1"), 169 | filepath.Join(scaffolingRoot, "d0/symlinks/toD1/f2"), 170 | filepath.Join(scaffolingRoot, "d0/symlinks/toF1"), 171 | } 172 | 173 | ensureStringSlicesMatch(t, actual, expected) 174 | }) 175 | } 176 | 177 | func TestWalkFollowSymbolicLinks(t *testing.T) { 178 | var actual []string 179 | var errorCallbackVisited bool 180 | 181 | err := Walk(filepath.Join(scaffolingRoot, "d0/symlinks"), &Options{ 182 | Callback: func(osPathname string, _ *Dirent) error { 183 | actual = append(actual, filepath.FromSlash(osPathname)) 184 | return nil 185 | }, 186 | ErrorCallback: func(osPathname string, err error) ErrorAction { 187 | if filepath.Base(osPathname) == "nothing" { 188 | errorCallbackVisited = true 189 | return SkipNode 190 | } 191 | return Halt 192 | }, 193 | FollowSymbolicLinks: true, 194 | }) 195 | 196 | ensureError(t, err) 197 | 198 | if got, want := errorCallbackVisited, true; got != want { 199 | t.Errorf("GOT: %v; WANT: %v", got, want) 200 | } 201 | 202 | expected := []string{ 203 | filepath.Join(scaffolingRoot, "d0/symlinks"), 204 | filepath.Join(scaffolingRoot, "d0/symlinks/d4"), 205 | filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSD1"), // chained symbolic link 206 | filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSD1/f2"), // chained symbolic link 207 | filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSF1"), // chained symbolic link 208 | filepath.Join(scaffolingRoot, "d0/symlinks/nothing"), 209 | filepath.Join(scaffolingRoot, "d0/symlinks/toAbs"), 210 | filepath.Join(scaffolingRoot, "d0/symlinks/toD1"), 211 | filepath.Join(scaffolingRoot, "d0/symlinks/toD1/f2"), 212 | filepath.Join(scaffolingRoot, "d0/symlinks/toF1"), 213 | } 214 | 215 | ensureStringSlicesMatch(t, actual, expected) 216 | } 217 | 218 | // While filepath.Walk will deliver the no access error to the regular callback, 219 | // godirwalk should deliver it first to the ErrorCallback handler, then take 220 | // action based on the return value of that callback function. 221 | func TestErrorCallback(t *testing.T) { 222 | t.Run("halt", func(t *testing.T) { 223 | var callbackVisited, errorCallbackVisited bool 224 | 225 | err := Walk(filepath.Join(scaffolingRoot, "d0/symlinks"), &Options{ 226 | Callback: func(osPathname string, dirent *Dirent) error { 227 | switch dirent.Name() { 228 | case "nothing": 229 | callbackVisited = true 230 | } 231 | return nil 232 | }, 233 | ErrorCallback: func(osPathname string, err error) ErrorAction { 234 | switch filepath.Base(osPathname) { 235 | case "nothing": 236 | errorCallbackVisited = true 237 | return Halt // Direct Walk to propagate error to caller 238 | } 239 | t.Fatalf("unexpected error callback for %s: %s", osPathname, err) 240 | return SkipNode 241 | }, 242 | FollowSymbolicLinks: true, 243 | }) 244 | 245 | ensureError(t, err, "nothing") // Ensure caller receives propagated access error 246 | if got, want := callbackVisited, true; got != want { 247 | t.Errorf("GOT: %v; WANT: %v", got, want) 248 | } 249 | if got, want := errorCallbackVisited, true; got != want { 250 | t.Errorf("GOT: %v; WANT: %v", got, want) 251 | } 252 | }) 253 | 254 | t.Run("skipnode", func(t *testing.T) { 255 | var callbackVisited, errorCallbackVisited bool 256 | 257 | err := Walk(filepath.Join(scaffolingRoot, "d0/symlinks"), &Options{ 258 | Callback: func(osPathname string, dirent *Dirent) error { 259 | switch dirent.Name() { 260 | case "nothing": 261 | callbackVisited = true 262 | } 263 | return nil 264 | }, 265 | ErrorCallback: func(osPathname string, err error) ErrorAction { 266 | switch filepath.Base(osPathname) { 267 | case "nothing": 268 | errorCallbackVisited = true 269 | return SkipNode // Direct Walk to ignore this error 270 | } 271 | t.Fatalf("unexpected error callback for %s: %s", osPathname, err) 272 | return Halt 273 | }, 274 | FollowSymbolicLinks: true, 275 | }) 276 | 277 | ensureError(t, err) // Ensure caller receives no access error 278 | if got, want := callbackVisited, true; got != want { 279 | t.Errorf("GOT: %v; WANT: %v", got, want) 280 | } 281 | if got, want := errorCallbackVisited, true; got != want { 282 | t.Errorf("GOT: %v; WANT: %v", got, want) 283 | } 284 | }) 285 | } 286 | 287 | // Invokes PostChildrenCallback for all directories and nothing else. 288 | func TestPostChildrenCallback(t *testing.T) { 289 | var actual []string 290 | 291 | err := Walk(filepath.Join(scaffolingRoot, "d0"), &Options{ 292 | Callback: func(_ string, _ *Dirent) error { return nil }, 293 | PostChildrenCallback: func(osPathname string, _ *Dirent) error { 294 | actual = append(actual, osPathname) 295 | return nil 296 | }, 297 | }) 298 | 299 | ensureError(t, err) 300 | 301 | expected := []string{ 302 | filepath.Join(scaffolingRoot, "d0"), 303 | filepath.Join(scaffolingRoot, "d0/d1"), 304 | filepath.Join(scaffolingRoot, "d0/skips"), 305 | filepath.Join(scaffolingRoot, "d0/skips/d2"), 306 | filepath.Join(scaffolingRoot, "d0/skips/d3"), 307 | filepath.Join(scaffolingRoot, "d0/skips/d3/skip"), 308 | filepath.Join(scaffolingRoot, "d0/symlinks"), 309 | filepath.Join(scaffolingRoot, "d0/symlinks/d4"), 310 | } 311 | 312 | ensureStringSlicesMatch(t, actual, expected) 313 | } 314 | 315 | const flameIterations = 10 316 | 317 | var goPrefix = filepath.Join(os.Getenv("GOPATH"), "src") 318 | 319 | func BenchmarkFilepath(b *testing.B) { 320 | if testing.Short() { 321 | b.Skip("Skipping benchmark using user's Go source directory") 322 | } 323 | for i := 0; i < b.N; i++ { 324 | _ = filepathWalk(b, goPrefix) 325 | } 326 | } 327 | 328 | func BenchmarkGodirwalk(b *testing.B) { 329 | if testing.Short() { 330 | b.Skip("Skipping benchmark using user's Go source directory") 331 | } 332 | for i := 0; i < b.N; i++ { 333 | _ = godirwalkWalk(b, goPrefix) 334 | } 335 | } 336 | 337 | func BenchmarkGodirwalkUnsorted(b *testing.B) { 338 | if testing.Short() { 339 | b.Skip("Skipping benchmark using user's Go source directory") 340 | } 341 | for i := 0; i < b.N; i++ { 342 | _ = godirwalkWalkUnsorted(b, goPrefix) 343 | } 344 | } 345 | 346 | func BenchmarkFlameGraphFilepath(b *testing.B) { 347 | for i := 0; i < flameIterations; i++ { 348 | _ = filepathWalk(b, goPrefix) 349 | } 350 | } 351 | 352 | func BenchmarkFlameGraphGodirwalk(b *testing.B) { 353 | for i := 0; i < flameIterations; i++ { 354 | _ = godirwalkWalk(b, goPrefix) 355 | } 356 | } 357 | --------------------------------------------------------------------------------