diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 99babe97..d086d0be 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -487,6 +487,10 @@ "Comment": "0.2.2-12-g0b12d6b", "Rev": "0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74" }, + { + "ImportPath": "github.com/karrick/godirwalk", + "Rev": "2de2192f9e35ce981c152a873ed943b93b79ced4" + }, { "ImportPath": "github.com/klauspost/crc32", "Rev": "a3b15ae34567abb20a22992b989cd76f48d09c47" diff --git a/container/common/helpers.go b/container/common/helpers.go index f5539b5d..2aab6acc 100644 --- a/container/common/helpers.go +++ b/container/common/helpers.go @@ -28,6 +28,7 @@ import ( "github.com/google/cadvisor/utils" "github.com/golang/glog" + "github.com/karrick/godirwalk" ) func DebugInfo(watches map[string][]string) map[string][]string { @@ -156,7 +157,12 @@ func readUInt64(dirpath string, file string) uint64 { // Lists all directories under "path" and outputs the results as children of "parent". func ListDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error { - entries, err := ioutil.ReadDir(dirpath) + buf := make([]byte, godirwalk.DefaultScratchBufferSize) + return listDirectories(dirpath, parent, recursive, output, buf) +} + +func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}, buf []byte) error { + dirents, err := godirwalk.ReadDirents(dirpath, buf) if err != nil { // Ignore if this hierarchy does not exist. if os.IsNotExist(err) { @@ -164,18 +170,21 @@ func ListDirectories(dirpath string, parent string, recursive bool, output map[s } return err } - for _, entry := range entries { + for _, dirent := range dirents { // We only grab directories. - if entry.IsDir() { - name := path.Join(parent, entry.Name()) - output[name] = struct{}{} + if !dirent.IsDir() { + continue + } + dirname := dirent.Name() - // List subcontainers if asked to. - if recursive { - err := ListDirectories(path.Join(dirpath, entry.Name()), name, true, output) - if err != nil { - return err - } + name := path.Join(parent, dirname) + output[name] = struct{}{} + + // List subcontainers if asked to. + if recursive { + err := listDirectories(path.Join(dirpath, dirname), name, true, output, buf) + if err != nil { + return err } } } diff --git a/container/common/helpers_test.go b/container/common/helpers_test.go new file mode 100644 index 00000000..071349a6 --- /dev/null +++ b/container/common/helpers_test.go @@ -0,0 +1,28 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" +) + +func BenchmarkListDirectories(b *testing.B) { + for i := 0; i < b.N; i++ { + output := make(map[string]struct{}) + if err := ListDirectories("/sys/fs/cgroup", "", true, output); err != nil { + b.Fatal(err) + } + } +} diff --git a/vendor/github.com/karrick/godirwalk/.gitignore b/vendor/github.com/karrick/godirwalk/.gitignore new file mode 100644 index 00000000..a1338d68 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/.gitignore @@ -0,0 +1,14 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ diff --git a/vendor/github.com/karrick/godirwalk/LICENSE b/vendor/github.com/karrick/godirwalk/LICENSE new file mode 100644 index 00000000..01ce194c --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2017, Karrick McDermott +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/karrick/godirwalk/README.md b/vendor/github.com/karrick/godirwalk/README.md new file mode 100644 index 00000000..4f9922fe --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/README.md @@ -0,0 +1,208 @@ +# godirwalk + +`godirwalk` is a library for traversing a directory tree on a file +system. + +In short, why do I use this library? + +1. It's faster than `filepath.Walk`. +1. It's more correct on Windows than `filepath.Walk`. +1. It's more easy to use than `filepath.Walk`. +1. It's more flexible than `filepath.Walk`. + +## Usage Example + +Additional examples are provided in the `examples/` subdirectory. + +This library will normalize the provided top level directory name +based on the os-specific path separator by calling `filepath.Clean` on +its first argument. However it always provides the pathname created by +using the correct os-specific path separator when invoking the +provided callback function. + +```Go + dirname := "some/directory/root" + err := godirwalk.Walk(dirname, &godirwalk.Options{ + Callback: func(osPathname string, de *godirwalk.Dirent) error { + fmt.Printf("%s %s\n", de.ModeType(), osPathname) + return nil + }, + Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc) + }) +``` + +This library not only provides functions for traversing a file system +directory tree, but also for obtaining a list of immediate descendants +of a particular directory, typically much more quickly than using +`os.ReadDir` or `os.ReadDirnames`. + +Documentation is available via +[![GoDoc](https://godoc.org/github.com/karrick/godirwalk?status.svg)](https://godoc.org/github.com/karrick/godirwalk). + +## Description + +Here's why I use `godirwalk` in preference to `filepath.Walk`, +`os.ReadDir`, and `os.ReadDirnames`. + +### It's faster than `filepath.Walk` + +When compared against `filepath.Walk` in benchmarks, it has been +observed to run between five and ten times the speed on darwin, at +speeds comparable to the that of the unix `find` utility; about twice +the speed on linux; and about four times the speed on Windows. + +How does it obtain this performance boost? It does less work to give +you nearly the same output. This library calls the same `syscall` +functions to do the work, but it makes fewer calls, does not throw +away information that it might need, and creates less memory churn +along the way by reusing the same scratch buffer rather than +reallocating a new buffer every time it reads data from the operating +system. + +While traversing a file system directory tree, `filepath.Walk` obtains +the list of immediate descendants of a directory, and throws away the +file system node type information provided by the operating system +that comes with the node's name. Then, immediately prior to invoking +the callback function, `filepath.Walk` invokes `os.Stat` for each +node, and passes the returned `os.FileInfo` information to the +callback. + +While the `os.FileInfo` information provided by `os.Stat` is extremely +helpful--and even includes the `os.FileMode` data--providing it +requires an additional system call for each node. + +Because most callbacks only care about what the node type is, this +library does not throw the type information away, but rather provides +that information to the callback function in the form of a +`os.FileMode` value. Note that the provided `os.FileMode` value that +this library provides only has the node type information, and does not +have the permission bits, sticky bits, or other information from the +file's mode. If the callback does care about a particular node's +entire `os.FileInfo` data structure, the callback can easiy invoke +`os.Stat` when needed, and only when needed. + +#### Benchmarks + +##### macOS + +```Bash +go test -bench=. +goos: darwin +goarch: amd64 +pkg: github.com/karrick/godirwalk +BenchmarkFilepathWalk-8 1 3001274570 ns/op +BenchmarkGoDirWalk-8 3 465573172 ns/op +BenchmarkFlameGraphFilepathWalk-8 1 6957916936 ns/op +BenchmarkFlameGraphGoDirWalk-8 1 4210582571 ns/op +PASS +ok github.com/karrick/godirwalk 16.822s +``` + +##### Linux + +```Bash +go test -bench=. +goos: linux +goarch: amd64 +pkg: github.com/karrick/godirwalk +BenchmarkFilepathWalk-12 1 1609189170 ns/op +BenchmarkGoDirWalk-12 5 211336628 ns/op +BenchmarkFlameGraphFilepathWalk-12 1 3968119932 ns/op +BenchmarkFlameGraphGoDirWalk-12 1 2139598998 ns/op +PASS +ok github.com/karrick/godirwalk 9.007s +``` + +### It's more correct on Windows than `filepath.Walk` + +I did not previously care about this either, but humor me. We all love +how we can write once and run everywhere. It is essential for the +language's adoption, growth, and success, that the software we create +can run unmodified on all architectures and operating systems +supported by Go. + +When the traversed file system has a logical loop caused by symbolic +links to directories, on unix `filepath.Walk` ignores symbolic links +and traverses the entire directory tree without error. On Windows +however, `filepath.Walk` will continue following directory symbolic +links, even though it is not supposed to, eventually causing +`filepath.Walk` to terminate early and return an error when the +pathname gets too long from concatenating endless loops of symbolic +links onto the pathname. This error comes from Windows, passes through +`filepath.Walk`, and to the upstream client running `filepath.Walk`. + +The takeaway is that behavior is different based on which platform +`filepath.Walk` is running. While this is clearly not intentional, +until it is fixed in the standard library, it presents a compatibility +problem. + +This library correctly identifies symbolic links that point to +directories and will only follow them when `FollowSymbolicLinks` is +set to true. Behavior on Windows and other operating systems is +identical. + +### It's more easy to use than `filepath.Walk` + +Since this library does not invoke `os.Stat` on every file system node +it encounters, there is no possible error event for the callback +function to filter on. The third argument in the `filepath.WalkFunc` +function signature to pass the error from `os.Stat` to the callback +function is no longer necessary, and thus eliminated from signature of +the callback function from this library. + +Also, `filepath.Walk` invokes the callback function with a solidus +delimited pathname regardless of the os-specific path separator. This +library invokes the callback function with the os-specific pathname +separator, obviating a call to `filepath.Clean` in the callback +function for each node prior to actually using the provided pathname. + +In other words, even on Windows, `filepath.Walk` will invoke the +callback with `some/path/to/foo.txt`, requiring well written clients +to perform pathname normalization for every file prior to working with +the specified file. In truth, many clients developed on unix and not +tested on Windows neglect this subtlety, and will result in software +bugs when running on Windows. This library would invoke the callback +function with `some\path\to\foo.txt` for the same file when running on +Windows, eliminating the need to normalize the pathname by the client, +and lessen the likelyhood that a client will work on unix but not on +Windows. + +### It's more flexible than `filepath.Walk` + +#### Configurable Handling of Symbolic Links + +The default behavior of this library is to ignore symbolic links to +directories when walking a directory tree, just like `filepath.Walk` +does. However, it does invoke the callback function with each node it +finds, including symbolic links. If a particular use case exists to +follow symbolic links when traversing a directory tree, this library +can be invoked in manner to do so, by setting the +`FollowSymbolicLinks` parameter to true. + +#### Configurable Sorting of Directory Children + +The default behavior of this library is to always sort the immediate +descendants of a directory prior to visiting each node, just like +`filepath.Walk` does. This is usually the desired behavior. However, +this does come at a performance penalty to sort the names when a +directory node has many entries. If a particular use case exists that +does not require sorting the directory's immediate descendants prior +to visiting its nodes, this library will skip the sorting step when +the `Unsorted` parameter is set to true. + +#### Configurable Post Children Callback + +This library provides upstream code with the ability to specify a +callback to be invoked for each directory after its children are +processed. This has been used to recursively delete empty directories +after traversing the file system in a more efficient manner. See the +`examples/clean-empties` directory for an example of this usage. + +#### Configurable Error Callback + +This library provides upstream code with the ability to specify a +callback to be invoked for errors that the operating system returns, +allowing the upstream code to determine the next course of action to +take, whether to halt walking the hierarchy, as it would do were no +error callback provided, or skip the node that caused the error. See +the `examples/walk-fast` directory for an example of this usage. diff --git a/vendor/github.com/karrick/godirwalk/dirent.go b/vendor/github.com/karrick/godirwalk/dirent.go new file mode 100644 index 00000000..5a277224 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/dirent.go @@ -0,0 +1,74 @@ +package godirwalk + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// Dirent stores the name and file system mode type of discovered file system +// entries. +type Dirent struct { + name string + modeType os.FileMode +} + +// NewDirent returns a newly initialized Dirent structure, or an error. This +// function does not follow symbolic links. +// +// This function is rarely used, as Dirent structures are provided by other +// functions in this library that read and walk directories. +func NewDirent(osPathname string) (*Dirent, error) { + fi, err := os.Lstat(osPathname) + if err != nil { + return nil, errors.Wrap(err, "cannot lstat") + } + return &Dirent{ + name: filepath.Base(osPathname), + modeType: fi.Mode() & os.ModeType, + }, nil +} + +// Name returns the basename of the file system entry. +func (de Dirent) Name() string { return de.name } + +// ModeType returns the mode bits that specify the file system node type. We +// could make our own enum-like data type for encoding the file type, but Go's +// runtime already gives us architecture independent file modes, as discussed in +// `os/types.go`: +// +// Go's runtime FileMode type has same definition on all systems, so that +// information about files can be moved from one system to another portably. +func (de Dirent) ModeType() os.FileMode { return de.modeType } + +// IsDir returns true if and only if the Dirent represents a file system +// directory. Note that on some operating systems, more than one file mode bit +// may be set for a node. For instance, on Windows, a symbolic link that points +// to a directory will have both the directory and the symbolic link bits set. +func (de Dirent) IsDir() bool { return de.modeType&os.ModeDir != 0 } + +// IsRegular returns true if and only if the Dirent represents a regular +// file. That is, it ensures that no mode type bits are set. +func (de Dirent) IsRegular() bool { return de.modeType&os.ModeType == 0 } + +// IsSymlink returns true if and only if the Dirent represents a file system +// symbolic link. Note that on some operating systems, more than one file mode +// bit may be set for a node. For instance, on Windows, a symbolic link that +// points to a directory will have both the directory and the symbolic link bits +// set. +func (de Dirent) IsSymlink() bool { return de.modeType&os.ModeSymlink != 0 } + +// Dirents represents a slice of Dirent pointers, which are sortable by +// name. This type satisfies the `sort.Interface` interface. +type Dirents []*Dirent + +// Len returns the count of Dirent structures in the slice. +func (l Dirents) Len() int { return len(l) } + +// Less returns true if and only if the Name of the element specified by the +// first index is lexicographically less than that of the second index. +func (l Dirents) Less(i, j int) bool { return l[i].name < l[j].name } + +// Swap exchanges the two Dirent entries specified by the two provided indexes. +func (l Dirents) Swap(i, j int) { l[i], l[j] = l[j], l[i] } diff --git a/vendor/github.com/karrick/godirwalk/doc.go b/vendor/github.com/karrick/godirwalk/doc.go new file mode 100644 index 00000000..0dfdabd4 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/doc.go @@ -0,0 +1,34 @@ +/* +Package godirwalk provides functions to read and traverse directory trees. + +In short, why do I use this library? + +* It's faster than `filepath.Walk`. + +* It's more correct on Windows than `filepath.Walk`. + +* It's more easy to use than `filepath.Walk`. + +* It's more flexible than `filepath.Walk`. + +USAGE + +This library will normalize the provided top level directory name based on the +os-specific path separator by calling `filepath.Clean` on its first +argument. However it always provides the pathname created by using the correct +os-specific path separator when invoking the provided callback function. + + dirname := "some/directory/root" + err := godirwalk.Walk(dirname, &godirwalk.Options{ + Callback: func(osPathname string, de *godirwalk.Dirent) error { + fmt.Printf("%s %s\n", de.ModeType(), osPathname) + return nil + }, + }) + +This library not only provides functions for traversing a file system directory +tree, but also for obtaining a list of immediate descendants of a particular +directory, typically much more quickly than using `os.ReadDir` or +`os.ReadDirnames`. +*/ +package godirwalk diff --git a/vendor/github.com/karrick/godirwalk/examples/clean-empties/main.go b/vendor/github.com/karrick/godirwalk/examples/clean-empties/main.go new file mode 100644 index 00000000..e6ea2b37 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/examples/clean-empties/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/karrick/godirwalk" + "github.com/pkg/errors" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "usage: %s dir1 [dir2 [dir3...]]\n", filepath.Base(os.Args[0])) + os.Exit(2) + } + + scratchBuffer := make([]byte, 64*1024) // allocate once and re-use each time + var count, total int + var err error + + for _, arg := range os.Args[1:] { + count, err = pruneEmptyDirectories(arg, scratchBuffer) + total += count + if err != nil { + break + } + } + + fmt.Fprintf(os.Stderr, "Removed %d empty directories\n", total) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) + os.Exit(1) + } +} + +func pruneEmptyDirectories(osDirname string, scratchBuffer []byte) (int, error) { + var count int + + err := godirwalk.Walk(osDirname, &godirwalk.Options{ + Unsorted: true, + ScratchBuffer: scratchBuffer, + Callback: func(_ string, _ *godirwalk.Dirent) error { + // no-op while diving in; all the fun happens in PostChildrenCallback + return nil + }, + PostChildrenCallback: func(osPathname string, _ *godirwalk.Dirent) error { + deChildren, err := godirwalk.ReadDirents(osPathname, scratchBuffer) + if err != nil { + return errors.Wrap(err, "cannot ReadDirents") + } + // NOTE: ReadDirents skips "." and ".." + if len(deChildren) > 0 { + return nil // this directory has children; no additional work here + } + if osPathname == osDirname { + return nil // do not remove provided root directory + } + err = os.Remove(osPathname) + if err == nil { + count++ + } + return err + }, + }) + + return count, err +} diff --git a/vendor/github.com/karrick/godirwalk/examples/walk-fast/main.go b/vendor/github.com/karrick/godirwalk/examples/walk-fast/main.go new file mode 100644 index 00000000..d38021f6 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/examples/walk-fast/main.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "os" + + "github.com/karrick/godirwalk" +) + +func main() { + dirname := "." + if len(os.Args) > 1 { + dirname = os.Args[1] + } + err := godirwalk.Walk(dirname, &godirwalk.Options{ + Callback: func(osPathname string, de *godirwalk.Dirent) error { + // fmt.Printf("%s %s\n", de.ModeType(), osPathname) + return nil + }, + ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { + // Your program may want to log the error somehow. + // fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) + + // For the purposes of this example, a simple SkipNode will suffice, + // although in reality perhaps additional logic might be called for. + return godirwalk.SkipNode + }, + Unsorted: true, // set true for faster yet non-deterministic enumeration (see godoc) + }) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } +} diff --git a/vendor/github.com/karrick/godirwalk/examples/walk-stdlib/main.go b/vendor/github.com/karrick/godirwalk/examples/walk-stdlib/main.go new file mode 100644 index 00000000..d486ee92 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/examples/walk-stdlib/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" +) + +func main() { + dirname := "." + if len(os.Args) > 1 { + dirname = os.Args[1] + } + err := filepath.Walk(dirname, func(osPathname string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // fmt.Printf("%s %s\n", info.Mode(), osPathname) + return nil + }) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } +} diff --git a/vendor/github.com/karrick/godirwalk/go.mod b/vendor/github.com/karrick/godirwalk/go.mod new file mode 100644 index 00000000..6b467a9e --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/go.mod @@ -0,0 +1,3 @@ +module github.com/karrick/godirwalk + +require github.com/pkg/errors v0.8.0 diff --git a/vendor/github.com/karrick/godirwalk/go.sum b/vendor/github.com/karrick/godirwalk/go.sum new file mode 100644 index 00000000..a31014d5 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/go.sum @@ -0,0 +1 @@ +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/vendor/github.com/karrick/godirwalk/readdir.go b/vendor/github.com/karrick/godirwalk/readdir.go new file mode 100644 index 00000000..2bba6897 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir.go @@ -0,0 +1,47 @@ +package godirwalk + +// ReadDirents returns a sortable slice of pointers to Dirent structures, each +// representing the file system name and mode type for one of the immediate +// descendant of the specified directory. If the specified directory is a +// symbolic link, it will be resolved. +// +// If an optional scratch buffer is provided that is at least one page of +// memory, it will be used when reading directory entries from the file system. +// +// children, err := godirwalk.ReadDirents(osDirname, nil) +// if err != nil { +// return nil, errors.Wrap(err, "cannot get list of directory children") +// } +// sort.Sort(children) +// for _, child := range children { +// fmt.Printf("%s %s\n", child.ModeType, child.Name) +// } +func ReadDirents(osDirname string, scratchBuffer []byte) (Dirents, error) { + return readdirents(osDirname, scratchBuffer) +} + +// ReadDirnames returns a slice of strings, representing the immediate +// descendants of the specified directory. If the specified directory is a +// symbolic link, it will be resolved. +// +// If an optional scratch buffer is provided that is at least one page of +// memory, it will be used when reading directory entries from the file system. +// +// Note that this function, depending on operating system, may or may not invoke +// the ReadDirents function, in order to prepare the list of immediate +// descendants. Therefore, if your program needs both the names and the file +// system mode types of descendants, it will always be faster to invoke +// ReadDirents directly, rather than calling this function, then looping over +// the results and calling os.Stat for each child. +// +// children, err := godirwalk.ReadDirnames(osDirname, nil) +// if err != nil { +// return nil, errors.Wrap(err, "cannot get list of directory children") +// } +// sort.Strings(children) +// for _, child := range children { +// fmt.Printf("%s\n", child) +// } +func ReadDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { + return readdirnames(osDirname, scratchBuffer) +} diff --git a/vendor/github.com/karrick/godirwalk/readdir_test.go b/vendor/github.com/karrick/godirwalk/readdir_test.go new file mode 100644 index 00000000..f5859f11 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir_test.go @@ -0,0 +1,124 @@ +package godirwalk + +import ( + "os" + "path/filepath" + "sort" + "testing" +) + +func TestReadDirents(t *testing.T) { + entries, err := ReadDirents("testdata", nil) + if err != nil { + t.Fatal(err) + } + + expected := Dirents{ + &Dirent{ + name: "dir1", + modeType: os.ModeDir, + }, + &Dirent{ + name: "dir2", + modeType: os.ModeDir, + }, + &Dirent{ + name: "dir3", + modeType: os.ModeDir, + }, + &Dirent{ + name: "dir4", + modeType: os.ModeDir, + }, + &Dirent{ + name: "dir5", + modeType: os.ModeDir, + }, + &Dirent{ + name: "dir6", + modeType: os.ModeDir, + }, + &Dirent{ + name: "file3", + modeType: 0, + }, + &Dirent{ + name: "symlinks", + modeType: os.ModeDir, + }, + } + + if got, want := len(entries), len(expected); got != want { + t.Fatalf("(GOT) %v; (WNT) %v", got, want) + } + + sort.Sort(entries) + sort.Sort(expected) + + for i := 0; i < len(entries); i++ { + if got, want := entries[i].name, expected[i].name; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + if got, want := entries[i].modeType, expected[i].modeType; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } +} + +func TestReadDirentsSymlinks(t *testing.T) { + const osDirname = "testdata/symlinks" + + // Because some platforms set multiple mode type bits, when we create the + // expected slice, we need to ensure the mode types are set appropriately. + var expected Dirents + for _, pathname := range []string{"dir-symlink", "file-symlink", "invalid-symlink"} { + info, err := os.Lstat(filepath.Join(osDirname, pathname)) + if err != nil { + t.Fatal(err) + } + expected = append(expected, &Dirent{name: pathname, modeType: info.Mode() & os.ModeType}) + } + + entries, err := ReadDirents(osDirname, nil) + if err != nil { + t.Fatal(err) + } + + if got, want := len(entries), len(expected); got != want { + t.Fatalf("(GOT) %v; (WNT) %v", got, want) + } + + sort.Sort(entries) + sort.Sort(expected) + + for i := 0; i < len(entries); i++ { + if got, want := entries[i].name, expected[i].name; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + if got, want := entries[i].modeType, expected[i].modeType; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } +} + +func TestReadDirnames(t *testing.T) { + entries, err := ReadDirnames("testdata", nil) + if err != nil { + t.Fatal(err) + } + + expected := []string{"dir1", "dir2", "dir3", "dir4", "dir5", "dir6", "file3", "symlinks"} + + if got, want := len(entries), len(expected); got != want { + t.Fatalf("(GOT) %v; (WNT) %v", got, want) + } + + sort.Strings(entries) + sort.Strings(expected) + + for i := 0; i < len(entries); i++ { + if got, want := entries[i], expected[i]; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } +} diff --git a/vendor/github.com/karrick/godirwalk/readdir_unix.go b/vendor/github.com/karrick/godirwalk/readdir_unix.go new file mode 100644 index 00000000..04a628f7 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir_unix.go @@ -0,0 +1,109 @@ +// +build darwin freebsd linux netbsd openbsd + +package godirwalk + +import ( + "os" + "path/filepath" + "syscall" + "unsafe" + + "github.com/pkg/errors" +) + +func readdirents(osDirname string, scratchBuffer []byte) (Dirents, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + var entries Dirents + + fd := int(dh.Fd()) + + if len(scratchBuffer) < MinimumScratchBufferSize { + scratchBuffer = make([]byte, DefaultScratchBufferSize) + } + + var de *syscall.Dirent + + for { + n, err := syscall.ReadDirent(fd, scratchBuffer) + if err != nil { + _ = dh.Close() // ignore potential error returned by Close + return nil, errors.Wrap(err, "cannot ReadDirent") + } + if n <= 0 { + break // end of directory reached + } + // Loop over the bytes returned by reading the directory entries. + buf := scratchBuffer[:n] + for len(buf) > 0 { + de = (*syscall.Dirent)(unsafe.Pointer(&buf[0])) // point entry to first syscall.Dirent in buffer + buf = buf[de.Reclen:] // advance buffer + + if inoFromDirent(de) == 0 { + continue // this item has been deleted, but not yet removed from directory + } + + nameSlice := nameFromDirent(de) + namlen := len(nameSlice) + if (namlen == 0) || (namlen == 1 && nameSlice[0] == '.') || (namlen == 2 && nameSlice[0] == '.' && nameSlice[1] == '.') { + continue // skip unimportant entries + } + osChildname := string(nameSlice) + + // Convert syscall constant, which is in purview of OS, to a + // constant defined by Go, assumed by this project to be stable. + var mode os.FileMode + switch de.Type { + case syscall.DT_REG: + // regular file + case syscall.DT_DIR: + mode = os.ModeDir + case syscall.DT_LNK: + mode = os.ModeSymlink + case syscall.DT_CHR: + mode = os.ModeDevice | os.ModeCharDevice + case syscall.DT_BLK: + mode = os.ModeDevice + case syscall.DT_FIFO: + mode = os.ModeNamedPipe + case syscall.DT_SOCK: + mode = os.ModeSocket + default: + // If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), + // then resolve actual mode by getting stat. + fi, err := os.Lstat(filepath.Join(osDirname, osChildname)) + if err != nil { + _ = dh.Close() // ignore potential error returned by Close + return nil, errors.Wrap(err, "cannot Stat") + } + // We only care about the bits that identify the type of a file + // system node, and can ignore append, exclusive, temporary, + // setuid, setgid, permission bits, and sticky bits, which are + // coincident to the bits that declare type of the file system + // node. + mode = fi.Mode() & os.ModeType + } + + entries = append(entries, &Dirent{name: osChildname, modeType: mode}) + } + } + if err = dh.Close(); err != nil { + return nil, err + } + return entries, nil +} + +func readdirnames(osDirname string, scratchBuffer []byte) ([]string, error) { + des, err := readdirents(osDirname, scratchBuffer) + if err != nil { + return nil, err + } + names := make([]string, len(des)) + for i, v := range des { + names[i] = v.name + } + return names, nil +} diff --git a/vendor/github.com/karrick/godirwalk/readdir_windows.go b/vendor/github.com/karrick/godirwalk/readdir_windows.go new file mode 100644 index 00000000..885a067a --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir_windows.go @@ -0,0 +1,54 @@ +package godirwalk + +import ( + "os" + + "github.com/pkg/errors" +) + +// The functions in this file are mere wrappers of what is already provided by +// standard library, in order to provide the same API as this library provides. +// +// The scratch buffer argument is ignored by this architecture. +// +// Please send PR or link to article if you know of a more performant way of +// enumerating directory contents and mode types on Windows. + +func readdirents(osDirname string, _ []byte) (Dirents, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + fileinfos, err := dh.Readdir(0) + if er := dh.Close(); err == nil { + err = er + } + if err != nil { + return nil, errors.Wrap(err, "cannot Readdir") + } + + entries := make(Dirents, len(fileinfos)) + for i, info := range fileinfos { + entries[i] = &Dirent{name: info.Name(), modeType: info.Mode() & os.ModeType} + } + + return entries, nil +} + +func readdirnames(osDirname string, _ []byte) ([]string, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + entries, err := dh.Readdirnames(0) + if er := dh.Close(); err == nil { + err = er + } + if err != nil { + return nil, errors.Wrap(err, "cannot Readdirnames") + } + + return entries, nil +} diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/file1a1 b/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/file1a1 new file mode 100644 index 00000000..3a39deef Binary files /dev/null and b/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/file1a1 differ diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/skip b/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/skip new file mode 100644 index 00000000..d6f2eeb7 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/skip @@ -0,0 +1 @@ +Wed Aug 23 12:46:58 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/z1a2 b/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/z1a2 new file mode 100644 index 00000000..6c923206 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir1/dir1a/z1a2 @@ -0,0 +1 @@ +Wed Aug 23 13:21:17 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir1/file1b b/vendor/github.com/karrick/godirwalk/testdata/dir1/file1b new file mode 100644 index 00000000..547c5056 Binary files /dev/null and b/vendor/github.com/karrick/godirwalk/testdata/dir1/file1b differ diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir2/file2a b/vendor/github.com/karrick/godirwalk/testdata/dir2/file2a new file mode 100644 index 00000000..7ec0d9d4 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir2/file2a @@ -0,0 +1 @@ +1503435080 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir2/skip/file2b1 b/vendor/github.com/karrick/godirwalk/testdata/dir2/skip/file2b1 new file mode 100644 index 00000000..792a23a3 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir2/skip/file2b1 @@ -0,0 +1 @@ +Wed Aug 23 13:20:25 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir2/z2c/file2c1 b/vendor/github.com/karrick/godirwalk/testdata/dir2/z2c/file2c1 new file mode 100644 index 00000000..cb30a539 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir2/z2c/file2c1 @@ -0,0 +1 @@ +Wed Aug 23 13:29:02 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir3/aaa.txt b/vendor/github.com/karrick/godirwalk/testdata/dir3/aaa.txt new file mode 100644 index 00000000..6abefa5d --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir3/aaa.txt @@ -0,0 +1 @@ +Mon Aug 28 11:38:14 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir3/skip b/vendor/github.com/karrick/godirwalk/testdata/dir3/skip new file mode 120000 index 00000000..e132db25 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir3/skip @@ -0,0 +1 @@ +zzz \ No newline at end of file diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir3/zzz/aaa.txt b/vendor/github.com/karrick/godirwalk/testdata/dir3/zzz/aaa.txt new file mode 100644 index 00000000..4f9a990d --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir3/zzz/aaa.txt @@ -0,0 +1 @@ +Mon Aug 28 11:39:03 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir4/aaa.txt b/vendor/github.com/karrick/godirwalk/testdata/dir4/aaa.txt new file mode 100644 index 00000000..016a1a8d --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir4/aaa.txt @@ -0,0 +1 @@ +Mon Aug 28 11:54:03 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir4/symlinkToDirectory b/vendor/github.com/karrick/godirwalk/testdata/dir4/symlinkToDirectory new file mode 120000 index 00000000..e132db25 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir4/symlinkToDirectory @@ -0,0 +1 @@ +zzz \ No newline at end of file diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir4/symlinkToFile b/vendor/github.com/karrick/godirwalk/testdata/dir4/symlinkToFile new file mode 120000 index 00000000..1fb6bbf5 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir4/symlinkToFile @@ -0,0 +1 @@ +aaa.txt \ No newline at end of file diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir4/zzz/aaa.txt b/vendor/github.com/karrick/godirwalk/testdata/dir4/zzz/aaa.txt new file mode 100644 index 00000000..069057d0 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir4/zzz/aaa.txt @@ -0,0 +1 @@ +Mon Aug 28 11:54:19 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir5/a1.txt b/vendor/github.com/karrick/godirwalk/testdata/dir5/a1.txt new file mode 100644 index 00000000..a66493e1 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir5/a1.txt @@ -0,0 +1 @@ +Wed Feb 14 21:18:51 UTC 2018 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir5/a2/a2a/a2a1.txt b/vendor/github.com/karrick/godirwalk/testdata/dir5/a2/a2a/a2a1.txt new file mode 100644 index 00000000..7e5f317c --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir5/a2/a2a/a2a1.txt @@ -0,0 +1 @@ +Wed Feb 14 21:20:10 UTC 2018 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir5/a2/a2b.txt b/vendor/github.com/karrick/godirwalk/testdata/dir5/a2/a2b.txt new file mode 100644 index 00000000..30e8f5fe --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir5/a2/a2b.txt @@ -0,0 +1 @@ +Wed Feb 14 21:19:02 UTC 2018 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir6/bravo.txt b/vendor/github.com/karrick/godirwalk/testdata/dir6/bravo.txt new file mode 100644 index 00000000..d74f74f8 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir6/bravo.txt @@ -0,0 +1 @@ +1518928732 diff --git a/vendor/github.com/karrick/godirwalk/testdata/dir6/code/123.txt b/vendor/github.com/karrick/godirwalk/testdata/dir6/code/123.txt new file mode 100644 index 00000000..19b271ae --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/dir6/code/123.txt @@ -0,0 +1 @@ +1518927922 diff --git a/vendor/github.com/karrick/godirwalk/testdata/file3 b/vendor/github.com/karrick/godirwalk/testdata/file3 new file mode 100644 index 00000000..10b597b3 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/file3 @@ -0,0 +1 @@ +Fri Aug 25 12:31:51 EDT 2017 diff --git a/vendor/github.com/karrick/godirwalk/testdata/symlinks/dir-symlink b/vendor/github.com/karrick/godirwalk/testdata/symlinks/dir-symlink new file mode 120000 index 00000000..777ebd01 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/symlinks/dir-symlink @@ -0,0 +1 @@ +../../testdata \ No newline at end of file diff --git a/vendor/github.com/karrick/godirwalk/testdata/symlinks/file-symlink b/vendor/github.com/karrick/godirwalk/testdata/symlinks/file-symlink new file mode 120000 index 00000000..c6ab7642 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/symlinks/file-symlink @@ -0,0 +1 @@ +../file3 \ No newline at end of file diff --git a/vendor/github.com/karrick/godirwalk/testdata/symlinks/invalid-symlink b/vendor/github.com/karrick/godirwalk/testdata/symlinks/invalid-symlink new file mode 120000 index 00000000..0edf4f30 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/testdata/symlinks/invalid-symlink @@ -0,0 +1 @@ +/non/existing/file \ No newline at end of file diff --git a/vendor/github.com/karrick/godirwalk/walk.go b/vendor/github.com/karrick/godirwalk/walk.go new file mode 100644 index 00000000..4c184ab8 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/walk.go @@ -0,0 +1,367 @@ +package godirwalk + +import ( + "os" + "path/filepath" + "sort" + + "github.com/pkg/errors" +) + +// DefaultScratchBufferSize specifies the size of the scratch buffer that will +// be allocated by Walk, ReadDirents, or ReadDirnames when a scratch buffer is +// not provided or the scratch buffer that is provided is smaller than +// MinimumScratchBufferSize bytes. This may seem like a large value; however, +// when a program intends to enumerate large directories, having a larger +// scratch buffer results in fewer operating system calls. +const DefaultScratchBufferSize = 64 * 1024 + +// MinimumScratchBufferSize specifies the minimum size of the scratch buffer +// that Walk, ReadDirents, and ReadDirnames will use when reading file entries +// from the operating system. It is initialized to the result from calling +// `os.Getpagesize()` during program startup. +var MinimumScratchBufferSize int + +func init() { + MinimumScratchBufferSize = os.Getpagesize() +} + +// Options provide parameters for how the Walk function operates. +type Options struct { + // ErrorCallback specifies a function to be invoked in the case of an error + // that could potentially be ignored while walking a file system + // hierarchy. When set to nil or left as its zero-value, any error condition + // causes Walk to immediately return the error describing what took + // place. When non-nil, this user supplied function is invoked with the OS + // pathname of the file system object that caused the error along with the + // error that took place. The return value of the supplied ErrorCallback + // function determines whether the error will cause Walk to halt immediately + // as it would were no ErrorCallback value provided, or skip this file + // system node yet continue on with the remaining nodes in the file system + // hierarchy. + // + // ErrorCallback is invoked both for errors that are returned by the + // runtime, and for errors returned by other user supplied callback + // functions. + ErrorCallback func(string, error) ErrorAction + + // FollowSymbolicLinks specifies whether Walk will follow symbolic links + // that refer to directories. When set to false or left as its zero-value, + // Walk will still invoke the callback function with symbolic link nodes, + // but if the symbolic link refers to a directory, it will not recurse on + // that directory. When set to true, Walk will recurse on symbolic links + // that refer to a directory. + FollowSymbolicLinks bool + + // Unsorted controls whether or not Walk will sort the immediate descendants + // of a directory by their relative names prior to visiting each of those + // entries. + // + // When set to false or left at its zero-value, Walk will get the list of + // immediate descendants of a particular directory, sort that list by + // lexical order of their names, and then visit each node in the list in + // sorted order. This will cause Walk to always traverse the same directory + // tree in the same order, however may be inefficient for directories with + // many immediate descendants. + // + // When set to true, Walk skips sorting the list of immediate descendants + // for a directory, and simply visits each node in the order the operating + // system enumerated them. This will be more fast, but with the side effect + // that the traversal order may be different from one invocation to the + // next. + Unsorted bool + + // Callback is a required function that Walk will invoke for every file + // system node it encounters. + Callback WalkFunc + + // PostChildrenCallback is an option function that Walk will invoke for + // every file system directory it encounters after its children have been + // processed. + PostChildrenCallback WalkFunc + + // ScratchBuffer is an optional byte slice to use as a scratch buffer for + // Walk to use when reading directory entries, to reduce amount of garbage + // generation. Not all architectures take advantage of the scratch + // buffer. If omitted or the provided buffer has fewer bytes than + // MinimumScratchBufferSize, then a buffer with DefaultScratchBufferSize + // bytes will be created and used once per Walk invocation. + ScratchBuffer []byte +} + +// ErrorAction defines a set of actions the Walk function could take based on +// the occurrence of an error while walking the file system. See the +// documentation for the ErrorCallback field of the Options structure for more +// information. +type ErrorAction int + +const ( + // Halt is the ErrorAction return value when the upstream code wants to halt + // the walk process when a runtime error takes place. It matches the default + // action the Walk function would take were no ErrorCallback provided. + Halt ErrorAction = iota + + // SkipNode is the ErrorAction return value when the upstream code wants to + // ignore the runtime error for the current file system node, skip + // processing of the node that caused the error, and continue walking the + // file system hierarchy with the remaining nodes. + SkipNode +) + +// WalkFunc is the type of the function called for each file system node visited +// by Walk. The pathname argument will contain the argument to Walk as a prefix; +// that is, if Walk is called with "dir", which is a directory containing the +// file "a", the provided WalkFunc will be invoked with the argument "dir/a", +// using the correct os.PathSeparator for the Go Operating System architecture, +// GOOS. The directory entry argument is a pointer to a Dirent for the node, +// providing access to both the basename and the mode type of the file system +// node. +// +// If an error is returned by the Callback or PostChildrenCallback functions, +// and no ErrorCallback function is provided, processing stops. If an +// ErrorCallback function is provided, then it is invoked with the OS pathname +// of the node that caused the error along along with the error. The return +// value of the ErrorCallback function determines whether to halt processing, or +// skip this node and continue processing remaining file system nodes. +// +// The exception is when the function returns the special value +// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a +// directory, Walk skips the directory's contents entirely. If the function +// returns filepath.SkipDir when invoked on a non-directory file system node, +// Walk skips the remaining files in the containing directory. Note that any +// supplied ErrorCallback function is not invoked with filepath.SkipDir when the +// Callback or PostChildrenCallback functions return that special value. +type WalkFunc func(osPathname string, directoryEntry *Dirent) error + +// Walk walks the file tree rooted at the specified directory, calling the +// specified callback function for each file system node in the tree, including +// root, symbolic links, and other node types. The nodes are walked in lexical +// order, which makes the output deterministic but means that for very large +// directories this function can be inefficient. +// +// This function is often much faster than filepath.Walk because it does not +// invoke os.Stat for every node it encounters, but rather obtains the file +// system node type when it reads the parent directory. +// +// If a runtime error occurs, either from the operating system or from the +// upstream Callback or PostChildrenCallback functions, processing typically +// halts. However, when an ErrorCallback function is provided in the provided +// Options structure, that function is invoked with the error along with the OS +// pathname of the file system node that caused the error. The ErrorCallback +// function's return value determines the action that Walk will then take. +// +// func main() { +// dirname := "." +// if len(os.Args) > 1 { +// dirname = os.Args[1] +// } +// err := godirwalk.Walk(dirname, &godirwalk.Options{ +// Callback: func(osPathname string, de *godirwalk.Dirent) error { +// fmt.Printf("%s %s\n", de.ModeType(), osPathname) +// return nil +// }, +// ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { +// // Your program may want to log the error somehow. +// fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) +// +// // For the purposes of this example, a simple SkipNode will suffice, +// // although in reality perhaps additional logic might be called for. +// return godirwalk.SkipNode +// }, +// }) +// if err != nil { +// fmt.Fprintf(os.Stderr, "%s\n", err) +// os.Exit(1) +// } +// } +func Walk(pathname string, options *Options) error { + pathname = filepath.Clean(pathname) + + var fi os.FileInfo + var err error + + if options.FollowSymbolicLinks { + fi, err = os.Stat(pathname) + if err != nil { + return errors.Wrap(err, "cannot Stat") + } + } else { + fi, err = os.Lstat(pathname) + if err != nil { + return errors.Wrap(err, "cannot Lstat") + } + } + + mode := fi.Mode() + if mode&os.ModeDir == 0 { + return errors.Errorf("cannot Walk non-directory: %s", pathname) + } + + dirent := &Dirent{ + name: filepath.Base(pathname), + modeType: mode & os.ModeType, + } + + // If ErrorCallback is nil, set to a default value that halts the walk + // process on all operating system errors. This is done to allow error + // handling to be more succinct in the walk code. + if options.ErrorCallback == nil { + options.ErrorCallback = defaultErrorCallback + } + + if len(options.ScratchBuffer) < MinimumScratchBufferSize { + options.ScratchBuffer = make([]byte, DefaultScratchBufferSize) + } + + err = walk(pathname, dirent, options) + if err == filepath.SkipDir { + return nil // silence SkipDir for top level + } + return err +} + +// defaultErrorCallback always returns Halt because if the upstream code did not +// provide an ErrorCallback function, walking the file system hierarchy ought to +// halt upon any operating system error. +func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt } + +// walk recursively traverses the file system node specified by pathname and the +// Dirent. +func walk(osPathname string, dirent *Dirent, options *Options) error { + err := options.Callback(osPathname, dirent) + if err != nil { + if err == filepath.SkipDir { + return err + } + err = errors.Wrap(err, "Callback") // wrap potential errors returned by callback + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + // On some platforms, an entry can have more than one mode type bit set. + // For instance, it could have both the symlink bit and the directory bit + // set indicating it's a symlink to a directory. + if dirent.IsSymlink() { + if !options.FollowSymbolicLinks { + return nil + } + // Only need to Stat entry if platform did not already have os.ModeDir + // set, such as would be the case for unix like operating systems. (This + // guard eliminates extra os.Stat check on Windows.) + if !dirent.IsDir() { + referent, err := os.Readlink(osPathname) + if err != nil { + err = errors.Wrap(err, "cannot Readlink") + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + var osp string + if filepath.IsAbs(referent) { + osp = referent + } else { + osp = filepath.Join(filepath.Dir(osPathname), referent) + } + + fi, err := os.Stat(osp) + if err != nil { + err = errors.Wrap(err, "cannot Stat") + if action := options.ErrorCallback(osp, err); action == SkipNode { + return nil + } + return err + } + dirent.modeType = fi.Mode() & os.ModeType + } + } + + if !dirent.IsDir() { + return nil + } + + // If get here, then specified pathname refers to a directory. + deChildren, err := ReadDirents(osPathname, options.ScratchBuffer) + if err != nil { + err = errors.Wrap(err, "cannot ReadDirents") + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + if !options.Unsorted { + sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted + } + + for _, deChild := range deChildren { + osChildname := filepath.Join(osPathname, deChild.name) + err = walk(osChildname, deChild, options) + if err != nil { + if err != filepath.SkipDir { + return err + } + // If received skipdir on a directory, stop processing that + // directory, but continue to its siblings. If received skipdir on a + // non-directory, stop processing remaining siblings. + if deChild.IsSymlink() { + // Only need to Stat entry if platform did not already have + // os.ModeDir set, such as would be the case for unix like + // operating systems. (This guard eliminates extra os.Stat check + // on Windows.) + if !deChild.IsDir() { + // Resolve symbolic link referent to determine whether node + // is directory or not. + referent, err := os.Readlink(osChildname) + if err != nil { + err = errors.Wrap(err, "cannot Readlink") + if action := options.ErrorCallback(osChildname, err); action == SkipNode { + continue // with next child + } + return err + } + + var osp string + if filepath.IsAbs(referent) { + osp = referent + } else { + osp = filepath.Join(osPathname, referent) + } + + fi, err := os.Stat(osp) + if err != nil { + err = errors.Wrap(err, "cannot Stat") + if action := options.ErrorCallback(osp, err); action == SkipNode { + continue // with next child + } + return err + } + deChild.modeType = fi.Mode() & os.ModeType + } + } + if !deChild.IsDir() { + // If not directory, return immediately, thus skipping remainder + // of siblings. + return nil + } + } + } + + if options.PostChildrenCallback == nil { + return nil + } + + err = options.PostChildrenCallback(osPathname, dirent) + if err == nil || err == filepath.SkipDir { + return err + } + + err = errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by callback + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err +} diff --git a/vendor/github.com/karrick/godirwalk/walk_test.go b/vendor/github.com/karrick/godirwalk/walk_test.go new file mode 100644 index 00000000..d00057a5 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/walk_test.go @@ -0,0 +1,310 @@ +package godirwalk_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/karrick/godirwalk" +) + +const testScratchBufferSize = 16 * 1024 + +func helperFilepathWalk(tb testing.TB, osDirname string) []string { + var entries []string + err := filepath.Walk(osDirname, func(osPathname string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Name() == "skip" { + return filepath.SkipDir + } + // filepath.Walk invokes callback function with a slashed version of the + // pathname, while godirwalk invokes callback function with the + // os-specific pathname separator. + entries = append(entries, filepath.ToSlash(osPathname)) + return nil + }) + if err != nil { + tb.Fatal(err) + } + return entries +} + +func helperGodirwalkWalk(tb testing.TB, osDirname string) []string { + var entries []string + err := godirwalk.Walk(osDirname, &godirwalk.Options{ + Callback: func(osPathname string, dirent *godirwalk.Dirent) error { + if dirent.Name() == "skip" { + return filepath.SkipDir + } + // filepath.Walk invokes callback function with a slashed version of + // the pathname, while godirwalk invokes callback function with the + // os-specific pathname separator. + entries = append(entries, filepath.ToSlash(osPathname)) + return nil + }, + ScratchBuffer: make([]byte, testScratchBufferSize), + }) + if err != nil { + tb.Fatal(err) + } + return entries +} + +func symlinkAbs(oldname, newname string) error { + absDir, err := filepath.Abs(oldname) + if err != nil { + return err + } + return os.Symlink(absDir, newname) +} + +func TestWalkSkipDir(t *testing.T) { + // Ensure the results from calling filepath.Walk exactly match the results + // for calling this library's walk function. + + test := func(t *testing.T, osDirname string) { + expected := helperFilepathWalk(t, osDirname) + actual := helperGodirwalkWalk(t, osDirname) + + if got, want := len(actual), len(expected); got != want { + t.Fatalf("\n(GOT)\n\t%#v\n(WNT)\n\t%#v", actual, expected) + } + + for i := 0; i < len(actual); i++ { + if got, want := actual[i], expected[i]; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } + } + + // Test cases for encountering the filepath.SkipDir error at different times + // from the call. + + t.Run("SkipFileAtRoot", func(t *testing.T) { + test(t, "testdata/dir1/dir1a") + }) + + t.Run("SkipFileUnderRoot", func(t *testing.T) { + test(t, "testdata/dir1") + }) + + t.Run("SkipDirAtRoot", func(t *testing.T) { + test(t, "testdata/dir2/skip") + }) + + t.Run("SkipDirUnderRoot", func(t *testing.T) { + test(t, "testdata/dir2") + }) + + t.Run("SkipDirOnSymlink", func(t *testing.T) { + osDirname := "testdata/dir3" + actual := helperGodirwalkWalk(t, osDirname) + + expected := []string{ + "testdata/dir3", + "testdata/dir3/aaa.txt", + "testdata/dir3/zzz", + "testdata/dir3/zzz/aaa.txt", + } + + if got, want := len(actual), len(expected); got != want { + t.Fatalf("\n(GOT)\n\t%#v\n(WNT)\n\t%#v", actual, expected) + } + + for i := 0; i < len(actual); i++ { + if got, want := actual[i], expected[i]; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } + }) +} + +func TestWalkFollowSymbolicLinksFalse(t *testing.T) { + const ( + osDirname = "testdata/dir4" + symlink = "testdata/dir4/symlinkToAbsDirectory" + ) + + if err := symlinkAbs("testdata/dir4/zzz", symlink); err != nil { + t.Fatal(err) + } + + defer func() { + if err := os.Remove(symlink); err != nil { + t.Error(err) + } + }() + + var actual []string + err := godirwalk.Walk(osDirname, &godirwalk.Options{ + Callback: func(osPathname string, dirent *godirwalk.Dirent) error { + if dirent.Name() == "skip" { + return filepath.SkipDir + } + // filepath.Walk invokes callback function with a slashed version of + // the pathname, while godirwalk invokes callback function with the + // os-specific pathname separator. + actual = append(actual, filepath.ToSlash(osPathname)) + return nil + }, + }) + if err != nil { + t.Fatal(err) + } + + expected := []string{ + "testdata/dir4", + "testdata/dir4/aaa.txt", + "testdata/dir4/symlinkToAbsDirectory", + "testdata/dir4/symlinkToDirectory", + "testdata/dir4/symlinkToFile", + "testdata/dir4/zzz", + "testdata/dir4/zzz/aaa.txt", + } + + if got, want := len(actual), len(expected); got != want { + t.Fatalf("\n(GOT)\n\t%#v\n(WNT)\n\t%#v", actual, expected) + } + + for i := 0; i < len(actual); i++ { + if got, want := actual[i], expected[i]; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } +} + +func TestWalkFollowSymbolicLinksTrue(t *testing.T) { + const ( + osDirname = "testdata/dir4" + symlink = "testdata/dir4/symlinkToAbsDirectory" + ) + + if err := symlinkAbs("testdata/dir4/zzz", symlink); err != nil { + t.Fatal(err) + } + + defer func() { + if err := os.Remove(symlink); err != nil { + t.Error(err) + } + }() + + var actual []string + err := godirwalk.Walk(osDirname, &godirwalk.Options{ + FollowSymbolicLinks: true, + Callback: func(osPathname string, dirent *godirwalk.Dirent) error { + if dirent.Name() == "skip" { + return filepath.SkipDir + } + // filepath.Walk invokes callback function with a slashed version of + // the pathname, while godirwalk invokes callback function with the + // os-specific pathname separator. + actual = append(actual, filepath.ToSlash(osPathname)) + return nil + }, + }) + if err != nil { + t.Fatal(err) + } + + expected := []string{ + "testdata/dir4", + "testdata/dir4/aaa.txt", + "testdata/dir4/symlinkToAbsDirectory", + "testdata/dir4/symlinkToAbsDirectory/aaa.txt", + "testdata/dir4/symlinkToDirectory", + "testdata/dir4/symlinkToDirectory/aaa.txt", + "testdata/dir4/symlinkToFile", + "testdata/dir4/zzz", + "testdata/dir4/zzz/aaa.txt", + } + + if got, want := len(actual), len(expected); got != want { + t.Fatalf("\n(GOT)\n\t%#v\n(WNT)\n\t%#v", actual, expected) + } + + for i := 0; i < len(actual); i++ { + if got, want := actual[i], expected[i]; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } +} + +func TestPostChildrenCallback(t *testing.T) { + const osDirname = "testdata/dir5" + + var actual []string + + err := godirwalk.Walk(osDirname, &godirwalk.Options{ + ScratchBuffer: make([]byte, testScratchBufferSize), + Callback: func(osPathname string, _ *godirwalk.Dirent) error { + t.Logf("walk in: %s", osPathname) + return nil + }, + PostChildrenCallback: func(osPathname string, de *godirwalk.Dirent) error { + t.Logf("walk out: %s", osPathname) + actual = append(actual, osPathname) + return nil + }, + }) + if err != nil { + t.Errorf("(GOT): %v; (WNT): %v", err, nil) + } + + expected := []string{ + "testdata/dir5/a2/a2a", + "testdata/dir5/a2", + "testdata/dir5", + } + + if got, want := len(actual), len(expected); got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + + for i := 0; i < len(actual); i++ { + if i >= len(expected) { + t.Fatalf("(GOT) %v; (WNT): %v", actual[i], nil) + } + if got, want := actual[i], expected[i]; got != want { + t.Errorf("(GOT) %v; (WNT) %v", got, want) + } + } +} + +var goPrefix = filepath.Join(os.Getenv("GOPATH"), "src") + +func BenchmarkFilepathWalk(b *testing.B) { + if testing.Short() { + b.Skip("Skipping benchmark using user's Go source directory") + } + + for i := 0; i < b.N; i++ { + _ = helperFilepathWalk(b, goPrefix) + } +} + +func BenchmarkGoDirWalk(b *testing.B) { + if testing.Short() { + b.Skip("Skipping benchmark using user's Go source directory") + } + + for i := 0; i < b.N; i++ { + _ = helperGodirwalkWalk(b, goPrefix) + } +} + +const flameIterations = 10 + +func BenchmarkFlameGraphFilepathWalk(b *testing.B) { + for i := 0; i < flameIterations; i++ { + _ = helperFilepathWalk(b, goPrefix) + } +} + +func BenchmarkFlameGraphGoDirWalk(b *testing.B) { + for i := 0; i < flameIterations; i++ { + _ = helperGodirwalkWalk(b, goPrefix) + } +} diff --git a/vendor/github.com/karrick/godirwalk/withFileno.go b/vendor/github.com/karrick/godirwalk/withFileno.go new file mode 100644 index 00000000..1dc04a71 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withFileno.go @@ -0,0 +1,9 @@ +// +build dragonfly freebsd openbsd netbsd + +package godirwalk + +import "syscall" + +func inoFromDirent(de *syscall.Dirent) uint64 { + return uint64(de.Fileno) +} diff --git a/vendor/github.com/karrick/godirwalk/withIno.go b/vendor/github.com/karrick/godirwalk/withIno.go new file mode 100644 index 00000000..47fc1254 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withIno.go @@ -0,0 +1,9 @@ +// +build darwin linux + +package godirwalk + +import "syscall" + +func inoFromDirent(de *syscall.Dirent) uint64 { + return de.Ino +} diff --git a/vendor/github.com/karrick/godirwalk/withNamlen.go b/vendor/github.com/karrick/godirwalk/withNamlen.go new file mode 100644 index 00000000..46a4af50 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withNamlen.go @@ -0,0 +1,29 @@ +// +build darwin dragonfly freebsd netbsd openbsd + +package godirwalk + +import ( + "reflect" + "syscall" + "unsafe" +) + +func nameFromDirent(de *syscall.Dirent) []byte { + // Because this GOOS' syscall.Dirent provides a Namlen field that says how + // long the name is, this function does not need to search for the NULL + // byte. + ml := int(de.Namlen) + + // Convert syscall.Dirent.Name, which is array of int8, to []byte, by + // overwriting Cap, Len, and Data slice header fields to values from + // syscall.Dirent fields. Setting the Cap, Len, and Data field values for + // the slice header modifies what the slice header points to, and in this + // case, the name buffer. + var name []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) + sh.Cap = ml + sh.Len = ml + sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) + + return name +} diff --git a/vendor/github.com/karrick/godirwalk/withoutNamlen.go b/vendor/github.com/karrick/godirwalk/withoutNamlen.go new file mode 100644 index 00000000..dcf9f3a9 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withoutNamlen.go @@ -0,0 +1,36 @@ +// +build nacl linux solaris + +package godirwalk + +import ( + "bytes" + "reflect" + "syscall" + "unsafe" +) + +func nameFromDirent(de *syscall.Dirent) []byte { + // Because this GOOS' syscall.Dirent does not provide a field that specifies + // the name length, this function must first calculate the max possible name + // length, and then search for the NULL byte. + ml := int(uint64(de.Reclen) - uint64(unsafe.Offsetof(syscall.Dirent{}.Name))) + + // Convert syscall.Dirent.Name, which is array of int8, to []byte, by + // overwriting Cap, Len, and Data slice header fields to values from + // syscall.Dirent fields. Setting the Cap, Len, and Data field values for + // the slice header modifies what the slice header points to, and in this + // case, the name buffer. + var name []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) + sh.Cap = ml + sh.Len = ml + sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) + + if index := bytes.IndexByte(name, 0); index >= 0 { + // Found NULL byte; set slice's cap and len accordingly. + sh.Cap = index + sh.Len = index + } + + return name +}