diff --git a/cmd/files.go b/cmd/files.go index b34bea6..352e8a0 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/charlievieth/fastwalk" "seedno.de/seednode/roulette/types" ) @@ -227,8 +228,7 @@ func hasSupportedFiles(path string, formats *types.Types) (bool, error) { } func pathCount(path string) (int, int, error) { - var directories = 0 - var files = 0 + var directories, files = 0, 0 nodes, err := os.ReadDir(path) if err != nil { @@ -243,7 +243,7 @@ func pathCount(path string) (int, int, error) { } } - return files, directories, nil + return directories, files, nil } func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats *scanStatsChannels, formats *types.Types) error { @@ -252,7 +252,7 @@ func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats errorChannel := make(chan error) done := make(chan bool, 1) - filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error { + fastwalk.Walk(nil, path, func(p string, info os.DirEntry, err error) error { switch { case !Recursive && info.IsDir() && p != path: return filepath.SkipDir @@ -284,7 +284,7 @@ func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats stats.filesMatched <- 1 }() case info.IsDir(): - files, directories, err := pathCount(p) + directories, files, err := pathCount(p) if err != nil { errorChannel <- err } diff --git a/cmd/root.go b/cmd/root.go index 41ddd5c..3221c5e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -12,7 +12,7 @@ import ( ) const ( - ReleaseVersion string = "2.0.1" + ReleaseVersion string = "2.1.0" ) var ( @@ -95,7 +95,7 @@ func init() { rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)") rootCmd.Flags().BoolVar(&Handlers, "handlers", false, "display registered handlers (for debugging)") rootCmd.Flags().BoolVar(&Images, "images", false, "enable support for image files") - rootCmd.Flags().BoolVarP(&Index, "index", "c", false, "generate index of supported file paths at startup") + rootCmd.Flags().BoolVar(&Index, "index", false, "generate index of supported file paths at startup") rootCmd.Flags().StringVar(&IndexFile, "index-file", "", "path to optional persistent index file") rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints") rootCmd.Flags().IntVar(&MaxDirScans, "max-directory-scans", 32, "number of directories to scan at once") diff --git a/go.mod b/go.mod index 32baa15..ebc2adf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/alecthomas/chroma/v2 v2.9.1 + github.com/charlievieth/fastwalk v1.0.1 github.com/julienschmidt/httprouter v1.3.0 github.com/klauspost/compress v1.17.0 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index 15d9482..c01e1c0 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/alecthomas/chroma/v2 v2.9.1 h1:0O3lTQh9FxazJ4BYE/MOi/vDGuHn7B+6Bu902N github.com/alecthomas/chroma/v2 v2.9.1/go.mod h1:4TQu7gdfuPjSh76j78ietmqh9LiurGF0EpseFXdKMBw= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/charlievieth/fastwalk v1.0.1 h1:jW01w8OCFdKS9JvAcnI+JHhWU/FuIEmNb24Ri9p7OVg= +github.com/charlievieth/fastwalk v1.0.1/go.mod h1:dryXgMJyGHbMrAmmnF0/EJNBbZaihlwcNud5IuGyogU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= @@ -13,6 +15,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/vendor/github.com/charlievieth/fastwalk/.gitignore b/vendor/github.com/charlievieth/fastwalk/.gitignore new file mode 100644 index 0000000..21efe58 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/.gitignore @@ -0,0 +1,2 @@ +/vendor +*.test diff --git a/vendor/github.com/charlievieth/fastwalk/LICENSE b/vendor/github.com/charlievieth/fastwalk/LICENSE new file mode 100644 index 0000000..6d18063 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2022 Charlie Vieth + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/charlievieth/fastwalk/Makefile b/vendor/github.com/charlievieth/fastwalk/Makefile new file mode 100644 index 0000000..9fc53ac --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/Makefile @@ -0,0 +1,60 @@ +.PHONY: test_build_darwin_arm64 +test_build_darwin_arm64: + GOOS=darwin GOARCH=arm64 go test -c -o /dev/null + +.PHONY: test_build_darwin_amd64 +test_build_darwin_amd64: + GOOS=darwin GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_linux_arm64 +test_build_linux_arm64: + GOOS=linux GOARCH=arm64 go test -c -o /dev/null + +.PHONY: test_build_linux_amd64 +test_build_linux_amd64: + GOOS=linux GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_windows_amd64 +test_build_windows_amd64: + GOOS=windows GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_freebsd_amd64 +test_build_freebsd_amd64: + GOOS=freebsd GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_openbsd_amd64 +test_build_openbsd_amd64: + GOOS=openbsd GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_netbsd_amd64 +test_build_netbsd_amd64: + GOOS=netbsd GOARCH=amd64 go test -c -o /dev/null + +# Test that we can build fastwalk on multiple platforms +.PHONY: test_build +test_build: test_build_darwin_arm64 test_build_darwin_amd64 \ + test_build_linux_arm64 test_build_linux_amd64 \ + test_build_windows_amd64 test_build_freebsd_amd64 \ + test_build_openbsd_amd64 test_build_netbsd_amd64 + +.PHONY: test +test: # runs all tests against the package with race detection and coverage percentage + @go test -race -cover ./... +ifeq "$(shell go env GOOS)" "darwin" + @go test -tags nogetdirentries -race -cover ./... +endif + +.PHONY: quick +quick: # runs all tests without coverage or the race detector + @go test ./... + +.PHONY: bench +bench: + @go test -run '^$' -bench . -benchmem ./... + +.PHONY: bench_comp +bench_comp: + @go run ./scripts/bench_comp.go + +.PHONY: all +all: test test_build diff --git a/vendor/github.com/charlievieth/fastwalk/README.md b/vendor/github.com/charlievieth/fastwalk/README.md new file mode 100644 index 0000000..9a7ba96 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/README.md @@ -0,0 +1,218 @@ +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](https://pkg.go.dev/github.com/charlievieth/fastwalk) +[![Test fastwalk on macOS](https://github.com/charlievieth/fastwalk/actions/workflows/macos.yml/badge.svg)](https://github.com/charlievieth/fastwalk/actions/workflows/macos.yml) +[![Test fastwalk on Linux](https://github.com/charlievieth/fastwalk/actions/workflows/linux.yml/badge.svg)](https://github.com/charlievieth/fastwalk/actions/workflows/linux.yml) +[![Test fastwalk on Windows](https://github.com/charlievieth/fastwalk/actions/workflows/windows.yml/badge.svg)](https://github.com/charlievieth/fastwalk/actions/workflows/windows.yml) + +# fastwalk + +Fast parallel directory traversal for Golang. + +Package fastwalk provides a fast parallel version of [`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc) +that is \~2x faster on macOS, \~4x faster on Linux, \~6x faster on Windows, +allocates 50% less memory, and requires 25% fewer memory allocations. +Additionally, it is \~4-5x faster than [godirwalk](https://github.com/karrick/godirwalk) +across OSes. + +Inspired by and based off of [golang.org/x/tools/internal/fastwalk](https://pkg.go.dev/golang.org/x/tools@v0.1.9/internal/fastwalk). + +## Features + +* Fast: multiple goroutines stat the filesystem and call the + [`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc) callback concurrently +* Safe symbolic link traversal ([`Config.Follow`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Config)) +* Same behavior and callback signature as [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) +* Wrapper functions are provided to ignore duplicate files and directories: + [`IgnoreDuplicateFiles()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#IgnoreDuplicateFiles) + and + [`IgnoreDuplicateDirs()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#IgnoreDuplicateDirs) +* Extensively tested on macOS, Linux, and Windows + +## Usage + +Usage is the same as [`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc), +but the [`walkFn`](https://pkg.go.dev/path/filepath@go1.17.7#WalkFunc) +argument to [`fastwalk.Walk`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk) +must be safe for concurrent use. + +Examples can be found in the [examples](./examples) directory. + + + +The below example is a very simple version of the POSIX +[find](https://pubs.opengroup.org/onlinepubs/007904975/utilities/find.html) utility: +```go +// fwfind is a an example program that is similar to POSIX find, +// but faster and worse (it's an example). +package main + +import ( + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/charlievieth/fastwalk" +) + +const usageMsg = `Usage: %[1]s [-L] [-name] [PATH...]: + +%[1]s is a poor replacement for the POSIX find utility + +` + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stdout, usageMsg, filepath.Base(os.Args[0])) + flag.PrintDefaults() + } + pattern := flag.String("name", "", "Pattern to match file names against.") + followLinks := flag.Bool("L", false, "Follow symbolic links") + flag.Parse() + + // If no paths are provided default to the current directory: "." + args := flag.Args() + if len(args) == 0 { + args = append(args, ".") + } + + // Follow links if the "-L" flag is provided + conf := fastwalk.Config{ + Follow: *followLinks, + } + + walkFn := func(path string, d fs.DirEntry, err error) error { + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", path, err) + return nil // returning the error stops iteration + } + if *pattern != "" { + if ok, err := filepath.Match(*pattern, d.Name()); !ok { + // invalid pattern (err != nil) or name does not match + return err + } + } + _, err = fmt.Println(path) + return err + } + for _, root := range args { + if err := fastwalk.Walk(&conf, root, walkFn); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", root, err) + os.Exit(1) + } + } +} +``` + +## Benchmarks + +Benchmarks were created using `go1.17.6` and can be generated with the `bench_comp` make target: +```sh +$ make bench_comp +``` + +### Darwin + +**Hardware:** +``` +goos: darwin +goarch: arm64 +cpu: Apple M1 Max +``` + +#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): +``` + filepath fastwalk delta +time/op 27.9ms ± 1% 13.0ms ± 1% -53.33% +alloc/op 4.33MB ± 0% 2.14MB ± 0% -50.55% +allocs/op 50.9k ± 0% 37.7k ± 0% -26.01% +``` + +#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): +``` + godirwalk fastwalk delta +time/op 58.5ms ± 3% 18.0ms ± 2% -69.30% +alloc/op 25.3MB ± 0% 2.1MB ± 0% -91.55% +allocs/op 57.6k ± 0% 37.7k ± 0% -34.59% +``` + +### Linux + +**Hardware:** +``` +goos: linux +goarch: amd64 +cpu: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz +drive: Samsung SSD 970 PRO 1TB +``` + +#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 10.1ms ± 2% 2.8ms ± 2% -72.83% +alloc/op 2.44MB ± 0% 1.70MB ± 0% -30.46% +allocs/op 47.2k ± 0% 36.9k ± 0% -21.80% +``` + +#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 13.7ms ±16% 2.8ms ± 2% -79.88% +alloc/op 7.48MB ± 0% 1.70MB ± 0% -77.34% +allocs/op 53.8k ± 0% 36.9k ± 0% -31.38% +``` + +### Windows + +**Hardware:** +``` +goos: windows +goarch: amd64 +pkg: github.com/charlievieth/fastwalk +cpu: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz +``` + +#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 88.0ms ± 1% 14.6ms ± 1% -83.47% +alloc/op 5.68MB ± 0% 6.76MB ± 0% +19.01% +allocs/op 69.6k ± 0% 90.4k ± 0% +29.87% +``` + +#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 87.4ms ± 1% 14.6ms ± 1% -83.34% +alloc/op 6.14MB ± 0% 6.76MB ± 0% +10.24% +allocs/op 100k ± 0% 90k ± 0% -9.59% +``` + +## Darwin: getdirentries64 + +The `nogetdirentries` build tag can be used to prevent `fastwalk` from using +and linking to the non-public `__getdirentries64` syscall. This is required +if an app using `fastwalk` is to be distributed via Apple's App Store (see +https://github.com/golang/go/issues/30933 for more details). When using +`__getdirentries64` is disabled, `fastwalk` will use `readdir_r` instead, +which is what the Go standard library uses for +[`os.ReadDir`](https://pkg.go.dev/os#ReadDir) and is about \~10% slower than +`__getdirentries64` +([benchmarks](https://github.com/charlievieth/fastwalk/blob/2e6a1b8a1ce88e578279e6e631b2129f7144ec87/fastwalk_darwin_test.go#L19-L57)). + +Example of how to build and test that your program is not linked to `__getdirentries64`: +```sh +# NOTE: the following only applies to darwin (aka macOS) + +# Build binary that imports fastwalk without linking to __getdirentries64. +$ go build -tags nogetdirentries -o YOUR_BINARY +# Test that __getdirentries64 is not linked (this should print no output). +$ ! otool -dyld_info YOUR_BINARY | grep -F getdirentries64 +``` + +There is a also a script [scripts/links2getdirentries.bash](scripts/links2getdirentries.bash) +that can be used to check if a program binary links to getdirentries. diff --git a/vendor/github.com/charlievieth/fastwalk/adapters.go b/vendor/github.com/charlievieth/fastwalk/adapters.go new file mode 100644 index 0000000..a7cec8a --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/adapters.go @@ -0,0 +1,105 @@ +package fastwalk + +import ( + "io/fs" + "os" + "path/filepath" +) + +func isDir(path string, d fs.DirEntry) bool { + if d.IsDir() { + return true + } + if d.Type()&os.ModeSymlink != 0 { + if fi, err := StatDirEntry(path, d); err == nil { + return fi.IsDir() + } + } + return false +} + +// IgnoreDuplicateDirs wraps fs.WalkDirFunc walkFn to make it follow symbolic +// links and ignore duplicate directories (if a symlink points to a directory +// that has already been traversed it is skipped). The walkFn is called for +// for skipped directories, but the directory is not traversed (this is +// required for error handling). +// +// The Config.Follow setting has no effect on the behavior of Walk when +// this wrapper is used. +// +// In most use cases, the returned fs.WalkDirFunc should not be reused between +// in another call to Walk. If it is reused, any previously visited file will +// be skipped. +// +// NOTE: The order of traversal is undefined. Given an "example" directory +// like the one below where "dir" is a directory and "smydir1" and "smydir2" +// are links to it, only one of "dir", "smydir1", or "smydir2" will be +// traversed, but which one is undefined. +// +// example +// ├── dir +// ├── smydir1 -> dir +// └── smydir2 -> dir +// +func IgnoreDuplicateDirs(walkFn fs.WalkDirFunc) fs.WalkDirFunc { + filter := NewEntryFilter() + return func(path string, d fs.DirEntry, err error) error { + // Call walkFn before checking the entry filter so that we + // don't record directories that are skipped with SkipDir. + err = walkFn(path, d, err) + if err != nil { + if err != filepath.SkipDir && isDir(path, d) { + filter.Entry(path, d) + } + return err + } + if isDir(path, d) { + if filter.Entry(path, d) { + return filepath.SkipDir + } + if d.Type() == os.ModeSymlink { + return ErrTraverseLink + } + } + return nil + } +} + +// IgnoreDuplicateFiles wraps walkFn so that symlinks are followed and duplicate +// files are ignored. If a symlink resolves to a file that has already been +// visited it will be skipped. +// +// In most use cases, the returned fs.WalkDirFunc should not be reused between +// in another call to Walk. If it is reused, any previously visited file will +// be skipped. +// +// This can significantly slow Walk as os.Stat() is called for each path +// (on Windows, os.Stat() is only needed for symlinks). +func IgnoreDuplicateFiles(walkFn fs.WalkDirFunc) fs.WalkDirFunc { + filter := NewEntryFilter() + return func(path string, d fs.DirEntry, err error) error { + // Skip all duplicate files, directories, and links + if filter.Entry(path, d) { + if isDir(path, d) { + return filepath.SkipDir + } + return nil + } + err = walkFn(path, d, err) + if err == nil && d.Type() == os.ModeSymlink && isDir(path, d) { + err = ErrTraverseLink + } + return err + } +} + +// IgnorePermissionErrors wraps walkFn so that permission errors are ignored. +// The returned fs.WalkDirFunc may be reused. +func IgnorePermissionErrors(walkFn fs.WalkDirFunc) fs.WalkDirFunc { + return func(path string, d fs.DirEntry, err error) error { + if err != nil && os.IsPermission(err) { + return nil + } + return walkFn(path, d, err) + } +} diff --git a/vendor/github.com/charlievieth/fastwalk/dirent.go b/vendor/github.com/charlievieth/fastwalk/dirent.go new file mode 100644 index 0000000..2f259e2 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/dirent.go @@ -0,0 +1,46 @@ +package fastwalk + +import ( + "io/fs" + "os" + "sync" + "sync/atomic" + "unsafe" +) + +type fileInfo struct { + once sync.Once + fs.FileInfo + err error +} + +func loadFileInfo(pinfo **fileInfo) *fileInfo { + ptr := (*unsafe.Pointer)(unsafe.Pointer(pinfo)) + fi := (*fileInfo)(atomic.LoadPointer(ptr)) + if fi == nil { + fi = &fileInfo{} + if !atomic.CompareAndSwapPointer( + (*unsafe.Pointer)(unsafe.Pointer(pinfo)), // adrr + nil, // old + unsafe.Pointer(fi), // new + ) { + fi = (*fileInfo)(atomic.LoadPointer(ptr)) + } + } + return fi +} + +// StatDirEntry returns the fs.FileInfo for the file or subdirectory described +// by the entry. If the entry is a symbolic link, StatDirEntry returns the +// fs.FileInfo for the file the line references (os.Stat). +// If fs.DirEntry de is a fastwalk.DirEntry it's Stat() method is used and the +// returned fs.FileInfo may be a previously cached result. +func StatDirEntry(path string, de fs.DirEntry) (fs.FileInfo, error) { + if de.Type()&os.ModeSymlink == 0 { + return de.Info() + } + if d, ok := de.(DirEntry); ok { + return d.Stat() + } + return os.Stat(path) +} diff --git a/vendor/github.com/charlievieth/fastwalk/dirent_portable.go b/vendor/github.com/charlievieth/fastwalk/dirent_portable.go new file mode 100644 index 0000000..bee77e8 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/dirent_portable.go @@ -0,0 +1,38 @@ +//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd) +// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd + +package fastwalk + +import ( + "io/fs" + "os" +) + +type portableDirent struct { + fs.DirEntry + path string + stat *fileInfo +} + +// TODO: cache the result of Stat +func (d *portableDirent) Stat() (fs.FileInfo, error) { + if d.DirEntry.Type()&os.ModeSymlink == 0 { + return d.DirEntry.Info() + } + stat := loadFileInfo(&d.stat) + stat.once.Do(func() { + stat.FileInfo, stat.err = os.Stat(d.path) + }) + return stat.FileInfo, stat.err +} + +func newDirEntry(dirName string, info fs.DirEntry) fs.DirEntry { + return &portableDirent{ + DirEntry: info, + path: dirName + string(os.PathSeparator) + info.Name(), + } +} + +func fileInfoToDirEntry(dirname string, fi fs.FileInfo) fs.DirEntry { + return newDirEntry(dirname, fs.FileInfoToDirEntry(fi)) +} diff --git a/vendor/github.com/charlievieth/fastwalk/dirent_unix.go b/vendor/github.com/charlievieth/fastwalk/dirent_unix.go new file mode 100644 index 0000000..a567589 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/dirent_unix.go @@ -0,0 +1,62 @@ +//go:build (aix || darwin || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd || solaris) && !appengine +// +build aix darwin dragonfly freebsd js,wasm linux netbsd openbsd solaris +// +build !appengine + +package fastwalk + +import ( + "io/fs" + "os" +) + +type unixDirent struct { + parent string + name string + typ os.FileMode + info *fileInfo + stat *fileInfo +} + +func (d *unixDirent) Name() string { return d.name } +func (d *unixDirent) IsDir() bool { return d.typ.IsDir() } +func (d *unixDirent) Type() os.FileMode { return d.typ } + +func (d *unixDirent) Info() (fs.FileInfo, error) { + info := loadFileInfo(&d.info) + info.once.Do(func() { + info.FileInfo, info.err = os.Lstat(d.parent + "/" + d.name) + }) + return info.FileInfo, info.err +} + +func (d *unixDirent) Stat() (fs.FileInfo, error) { + if d.typ&os.ModeSymlink == 0 { + return d.Info() + } + stat := loadFileInfo(&d.stat) + stat.once.Do(func() { + stat.FileInfo, stat.err = os.Stat(d.parent + "/" + d.name) + }) + return stat.FileInfo, stat.err +} + +func newUnixDirent(parent, name string, typ os.FileMode) *unixDirent { + return &unixDirent{ + parent: parent, + name: name, + typ: typ, + } +} + +func fileInfoToDirEntry(dirname string, fi fs.FileInfo) fs.DirEntry { + info := &fileInfo{ + FileInfo: fi, + } + info.once.Do(func() {}) + return &unixDirent{ + parent: dirname, + name: fi.Name(), + typ: fi.Mode().Type(), + info: info, + } +} diff --git a/vendor/github.com/charlievieth/fastwalk/entry_filter_portable.go b/vendor/github.com/charlievieth/fastwalk/entry_filter_portable.go new file mode 100644 index 0000000..e77ef6b --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/entry_filter_portable.go @@ -0,0 +1,38 @@ +//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd && !windows) +// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd,!windows + +package fastwalk + +import ( + "io/fs" + "path/filepath" + "sync" +) + +type EntryFilter struct { + // we assume most files have not been seen so + // no need for a RWMutex + mu sync.Mutex + seen map[string]struct{} +} + +func (e *EntryFilter) Entry(path string, _ fs.DirEntry) bool { + name, err := filepath.EvalSymlinks(path) + if err != nil { + return false + } + e.mu.Lock() + if e.seen == nil { + e.seen = make(map[string]struct{}, 128) + } + _, ok := e.seen[name] + if !ok { + e.seen[name] = struct{}{} + } + e.mu.Unlock() + return ok +} + +func NewEntryFilter() *EntryFilter { + return &EntryFilter{seen: make(map[string]struct{}, 128)} +} diff --git a/vendor/github.com/charlievieth/fastwalk/entry_filter_unix.go b/vendor/github.com/charlievieth/fastwalk/entry_filter_unix.go new file mode 100644 index 0000000..277ee2d --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/entry_filter_unix.go @@ -0,0 +1,61 @@ +//go:build (linux || darwin || freebsd || openbsd || netbsd || !windows) && !appengine +// +build linux darwin freebsd openbsd netbsd !windows +// +build !appengine + +package fastwalk + +import ( + "io/fs" + "sync" + "syscall" +) + +type fileKey struct { + Dev uint64 + Ino uint64 +} + +type entryMap struct { + mu sync.Mutex + keys map[fileKey]struct{} +} + +// An EntryFilter keeps track of visited directory entries and can be used to +// detect and avoid symlink loops or processing the same file twice. +type EntryFilter struct { + // Use an array of 8 to reduce lock contention. The entryMap is + // picked via the inode number. We don't take the device number + // into account because: we don't expect to see many of them and + // uniformly spreading the load isn't terribly beneficial here. + ents [8]entryMap +} + +// NewEntryFilter returns a new EntryFilter +func NewEntryFilter() *EntryFilter { + return new(EntryFilter) +} + +func (e *EntryFilter) seen(dev, ino uint64) (seen bool) { + m := &e.ents[ino%uint64(len(e.ents))] + m.mu.Lock() + if _, seen = m.keys[fileKey{dev, ino}]; !seen { + if m.keys == nil { + m.keys = make(map[fileKey]struct{}) + } + m.keys[fileKey{dev, ino}] = struct{}{} + } + m.mu.Unlock() + return seen +} + +// TODO: this name is confusing and should be fixed + +// Entry returns if path and fs.DirEntry have been seen before. +func (e *EntryFilter) Entry(path string, de fs.DirEntry) (seen bool) { + fi, err := StatDirEntry(path, de) + if err != nil { + return true // treat errors as duplicate files + } + stat := fi.Sys().(*syscall.Stat_t) + return e.seen(uint64(stat.Dev), uint64(stat.Ino)) +} diff --git a/vendor/github.com/charlievieth/fastwalk/entry_filter_windows.go b/vendor/github.com/charlievieth/fastwalk/entry_filter_windows.go new file mode 100644 index 0000000..e60f782 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/entry_filter_windows.go @@ -0,0 +1,157 @@ +//go:build windows && !appengine +// +build windows,!appengine + +package fastwalk + +import ( + "io/fs" + "os" + "path/filepath" + "sync" + "syscall" +) + +type fileKey struct { + VolumeSerialNumber uint32 + FileIndexHigh uint32 + FileIndexLow uint32 +} + +type EntryFilter struct { + mu sync.Mutex + seen map[fileKey]struct{} +} + +func NewEntryFilter() *EntryFilter { + return &EntryFilter{seen: make(map[fileKey]struct{}, 128)} +} + +func (e *EntryFilter) Entry(path string, _ fs.DirEntry) bool { + namep, err := syscall.UTF16PtrFromString(fixLongPath(path)) + if err != nil { + return false + } + + h, err := syscall.CreateFile(namep, 0, 0, nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return false + } + + var d syscall.ByHandleFileInformation + err = syscall.GetFileInformationByHandle(h, &d) + syscall.CloseHandle(h) + if err != nil { + return false + } + + key := fileKey{ + VolumeSerialNumber: d.VolumeSerialNumber, + FileIndexHigh: d.FileIndexHigh, + FileIndexLow: d.FileIndexLow, + } + + e.mu.Lock() + if e.seen == nil { + e.seen = make(map[fileKey]struct{}) + } + _, ok := e.seen[key] + if !ok { + e.seen[key] = struct{}{} + } + e.mu.Unlock() + + return ok +} + +func isAbs(path string) (b bool) { + v := filepath.VolumeName(path) + if v == "" { + return false + } + path = path[len(v):] + if path == "" { + return false + } + return os.IsPathSeparator(path[0]) +} + +// fixLongPath returns the extended-length (\\?\-prefixed) form of +// path when needed, in order to avoid the default 260 character file +// path limit imposed by Windows. If path is not easily converted to +// the extended-length form (for example, if path is a relative path +// or contains .. elements), or is short enough, fixLongPath returns +// path unmodified. +// +// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath +func fixLongPath(path string) string { + // Do nothing (and don't allocate) if the path is "short". + // Empirically (at least on the Windows Server 2013 builder), + // the kernel is arbitrarily okay with < 248 bytes. That + // matches what the docs above say: + // "When using an API to create a directory, the specified + // path cannot be so long that you cannot append an 8.3 file + // name (that is, the directory name cannot exceed MAX_PATH + // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. + // + // The MSDN docs appear to say that a normal path that is 248 bytes long + // will work; empirically the path must be less then 248 bytes long. + if len(path) < 248 { + // Don't fix. (This is how Go 1.7 and earlier worked, + // not automatically generating the \\?\ form) + return path + } + + // The extended form begins with \\?\, as in + // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt. + // The extended form disables evaluation of . and .. path + // elements and disables the interpretation of / as equivalent + // to \. The conversion here rewrites / to \ and elides + // . elements as well as trailing or duplicate separators. For + // simplicity it avoids the conversion entirely for relative + // paths or paths containing .. elements. For now, + // \\server\share paths are not converted to + // \\?\UNC\server\share paths because the rules for doing so + // are less well-specified. + if len(path) >= 2 && path[:2] == `\\` { + // Don't canonicalize UNC paths. + return path + } + if !isAbs(path) { + // Relative path + return path + } + + const prefix = `\\?` + + pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) + copy(pathbuf, prefix) + n := len(path) + r, w := 0, len(prefix) + for r < n { + switch { + case os.IsPathSeparator(path[r]): + // empty block + r++ + case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): + // /./ + r++ + case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): + // /../ is currently unhandled + return path + default: + pathbuf[w] = '\\' + w++ + for ; r < n && !os.IsPathSeparator(path[r]); r++ { + pathbuf[w] = path[r] + w++ + } + } + } + // A drive's root directory needs a trailing \ + if w == len(`\\?\c:`) { + pathbuf[w] = '\\' + w++ + } + return string(pathbuf[:w]) +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk.go b/vendor/github.com/charlievieth/fastwalk/fastwalk.go new file mode 100644 index 0000000..d30b2df --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk.go @@ -0,0 +1,394 @@ +// Package fastwalk provides a faster version of filepath.Walk for file system +// scanning tools. +package fastwalk + +/* + * This code borrows heavily from golang.org/x/tools/internal/fastwalk + * and as such the Go license can be found in the go.LICENSE file and + * is reproduced below: + * + * Copyright (c) 2009 The Go Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "sync" +) + +// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the +// symlink named in the call may be traversed. +var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory") + +// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the +// callback should not be called for any other files in the current directory. +// Child directories will still be traversed. +var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory") + +// SkipDir is used as a return value from WalkDirFuncs to indicate that +// the directory named in the call is to be skipped. It is not returned +// as an error by any function. +var SkipDir = fs.SkipDir + +// DefaultNumWorkers returns the default number of worker goroutines to use in +// fastwalk.Walk and is the value of runtime.GOMAXPROCS(-1) clamped to a range +// of 4 to 32 except on Darwin where it is either 4 (8 cores or less) or 6 +// (more than 8 cores). This is because Walk / IO performance on Darwin +// degrades with more concurrency. +// +// The optimal number for your workload may be lower or higher. The results +// of BenchmarkFastWalkNumWorkers benchmark may be informative. +func DefaultNumWorkers() int { + numCPU := runtime.GOMAXPROCS(-1) + if numCPU < 4 { + return 4 + } + // Darwin IO performance on APFS slows with more workers. + // Stat performance is best around 2-4 and file IO is best + // around 4-6. More workers only benefit CPU intensive tasks. + if runtime.GOOS == "darwin" { + if numCPU <= 8 { + return 4 + } + return 6 + } + if numCPU > 32 { + return 32 + } + return numCPU +} + +// DefaultConfig is the default Config used when none is supplied. +var DefaultConfig = Config{ + Follow: false, + NumWorkers: DefaultNumWorkers(), +} + +type Config struct { + // TODO: do we want to pass a sentinel error to WalkFunc if + // a symlink loop is detected? + + // Follow symbolic links ignoring directories that would lead + // to infinite loops; that is, entering a previously visited + // directory that is an ancestor of the last file encountered. + // + // The sentinel error ErrTraverseLink is ignored when Follow + // is true (this to prevent users from defeating the loop + // detection logic), but SkipDir and ErrSkipFiles are still + // respected. + Follow bool + + // Number of parallel workers to use. If NumWorkers if ≤ 0 then + // the greater of runtime.NumCPU() or 4 is used. + NumWorkers int +} + +// A DirEntry extends the fs.DirEntry interface to add a Stat() method +// that returns the result of calling os.Stat() on the underlying file. +// The results of Info() and Stat() are cached. +// +// The fs.DirEntry argument passed to the fs.WalkDirFunc by Walk is +// always a DirEntry. The only exception is the root directory with +// with Walk is called. +type DirEntry interface { + fs.DirEntry + + // Stat returns the FileInfo for the file or subdirectory described + // by the entry. The returned FileInfo may be from the time of the + // original directory read or from the time of the call to Stat. + // If the entry denotes a symbolic link, Stat reports the information + // about the target itself, not the link. + Stat() (fs.FileInfo, error) +} + +// Walk is a faster implementation of filepath.Walk. +// +// filepath.Walk's design necessarily calls os.Lstat on each file, even if +// the caller needs less info. Many tools need only the type of each file. +// On some platforms, this information is provided directly by the readdir +// system call, avoiding the need to stat each file individually. +// fastwalk_unix.go contains a fork of the syscall routines. +// +// See golang.org/issue/16399 +// +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +// +// If walkFn returns filepath.SkipDir, the directory is skipped. +// +// Unlike filepath.WalkDir: +// * File stat calls must be done by the user and should be done via +// the DirEntry argument to walkFn since it caches the results of +// Stat and Lstat. +// * The fs.DirEntry argument is always a fastwalk.DirEntry, which has +// a Stat() method that returns the result of calling os.Stat() on the +// file. The result of Stat() may be cached. +// * Multiple goroutines stat the filesystem concurrently. The provided +// walkFn must be safe for concurrent use. +// * Walk can follow symlinks if walkFn returns the ErrTraverseLink +// sentinel error. It is the walkFn's responsibility to prevent +// Walk from going into symlink cycles. +func Walk(conf *Config, root string, walkFn fs.WalkDirFunc) error { + if conf == nil { + dupe := DefaultConfig + conf = &dupe + } + fi, err := os.Lstat(root) + if err != nil { + return err + } + + // Make sure to wait for all workers to finish, otherwise + // walkFn could still be called after returning. This Wait call + // runs after close(e.donec) below. + var wg sync.WaitGroup + defer wg.Wait() + + numWorkers := conf.NumWorkers + if numWorkers <= 0 { + numWorkers = DefaultNumWorkers() + } + + w := &walker{ + fn: walkFn, + enqueuec: make(chan walkItem, numWorkers), // buffered for performance + workc: make(chan walkItem, numWorkers), // buffered for performance + donec: make(chan struct{}), + + // buffered for correctness & not leaking goroutines: + resc: make(chan error, numWorkers), + + follow: conf.Follow, + } + if w.follow { + if fi, err := os.Stat(root); err == nil { + w.ignoredDirs = append(w.ignoredDirs, fi) + } + } + + defer close(w.donec) + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go w.doWork(&wg) + } + + root = cleanRootPath(root) + todo := []walkItem{{dir: root, info: fileInfoToDirEntry(filepath.Dir(root), fi)}} + out := 0 + for { + workc := w.workc + var workItem walkItem + if len(todo) == 0 { + workc = nil + } else { + workItem = todo[len(todo)-1] + } + select { + case workc <- workItem: + todo = todo[:len(todo)-1] + out++ + case it := <-w.enqueuec: + todo = append(todo, it) + case err := <-w.resc: + out-- + if err != nil { + return err + } + if out == 0 && len(todo) == 0 { + // It's safe to quit here, as long as the buffered + // enqueue channel isn't also readable, which might + // happen if the worker sends both another unit of + // work and its result before the other select was + // scheduled and both w.resc and w.enqueuec were + // readable. + select { + case it := <-w.enqueuec: + todo = append(todo, it) + default: + return nil + } + } + } + } +} + +// doWork reads directories as instructed (via workc) and runs the +// user's callback function. +func (w *walker) doWork(wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case <-w.donec: + return + case it := <-w.workc: + select { + case <-w.donec: + return + case w.resc <- w.walk(it.dir, it.info, !it.callbackDone): + } + } + } +} + +type walker struct { + fn fs.WalkDirFunc + + donec chan struct{} // closed on fastWalk's return + workc chan walkItem // to workers + enqueuec chan walkItem // from workers + resc chan error // from workers + + ignoredDirs []os.FileInfo + follow bool +} + +type walkItem struct { + dir string + info fs.DirEntry + callbackDone bool // callback already called; don't do it again +} + +func (w *walker) enqueue(it walkItem) { + select { + case w.enqueuec <- it: + case <-w.donec: + } +} + +func (w *walker) shouldSkipDir(fi os.FileInfo) bool { + for _, ignored := range w.ignoredDirs { + if os.SameFile(ignored, fi) { + return true + } + } + return false +} + +func (w *walker) shouldTraverse(path string, de fs.DirEntry) bool { + // TODO: do we need to use filepath.EvalSymlinks() here? + ts, err := StatDirEntry(path, de) + if err != nil { + return false + } + if !ts.IsDir() { + return false + } + if w.shouldSkipDir(ts) { + return false + } + for { + parent := filepath.Dir(path) + if parent == path { + return true + } + parentInfo, err := os.Stat(parent) + if err != nil { + return false + } + if os.SameFile(ts, parentInfo) { + return false + } + path = parent + } +} + +func joinPaths(dir, base string) string { + // Handle the case where the root path argument to Walk is "/" + // without this the returned path is prefixed with "//". + if os.PathSeparator == '/' && dir == "/" { + return dir + base + } + return dir + string(os.PathSeparator) + base +} + +func (w *walker) onDirEnt(dirName, baseName string, de fs.DirEntry) error { + joined := joinPaths(dirName, baseName) + typ := de.Type() + if typ == os.ModeDir { + w.enqueue(walkItem{dir: joined, info: de}) + return nil + } + + err := w.fn(joined, de, nil) + if typ == os.ModeSymlink { + if err == ErrTraverseLink { + if !w.follow { + // Set callbackDone so we don't call it twice for both the + // symlink-as-symlink and the symlink-as-directory later: + w.enqueue(walkItem{dir: joined, info: de, callbackDone: true}) + return nil + } + err = nil // Ignore ErrTraverseLink when Follow is true. + } + if err == filepath.SkipDir { + // Permit SkipDir on symlinks too. + return nil + } + if err == nil && w.follow && w.shouldTraverse(joined, de) { + // Traverse symlink + w.enqueue(walkItem{dir: joined, info: de, callbackDone: true}) + } + } + return err +} + +func (w *walker) walk(root string, info fs.DirEntry, runUserCallback bool) error { + if runUserCallback { + err := w.fn(root, info, nil) + if err == filepath.SkipDir { + return nil + } + if err != nil { + return err + } + } + + err := readDir(root, w.onDirEnt) + if err != nil { + // Second call, to report ReadDir error. + return w.fn(root, info, err) + } + return nil +} + +func cleanRootPath(root string) string { + for i := len(root) - 1; i >= 0; i-- { + if !os.IsPathSeparator(root[i]) { + return root[:i+1] + } + } + if root != "" { + return root[0:1] // root is all path separators ("//") + } + return root +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_darwin.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_darwin.go new file mode 100644 index 0000000..f5deb7c --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_darwin.go @@ -0,0 +1,188 @@ +//go:build darwin && go1.13 && !appengine +// +build darwin,go1.13,!appengine + +package fastwalk + +import ( + "io/fs" + "os" + "sync" + "syscall" + "unsafe" +) + +//sys closedir(dir uintptr) (err error) +//sys readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno) + +func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + if useGetdirentries { + return readDir_Getdirentries(dirName, fn) + } else { + return readDir_Readir(dirName, fn) + } +} + +func readDir_Readir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + fd, err := opendir(dirName) + if err != nil { + return &os.PathError{Op: "opendir", Path: dirName, Err: err} + } + defer closedir(fd) //nolint:errcheck + + skipFiles := false + var dirent syscall.Dirent + var entptr *syscall.Dirent + for { + if errno := readdir_r(fd, &dirent, &entptr); errno != 0 { + if errno == syscall.EINTR { + continue + } + return &os.PathError{Op: "readdir", Path: dirName, Err: errno} + } + if entptr == nil { // EOF + break + } + if dirent.Ino == 0 { + continue + } + typ := dtToType(dirent.Type) + if skipFiles && typ.IsRegular() { + continue + } + name := (*[len(syscall.Dirent{}.Name)]byte)(unsafe.Pointer(&dirent.Name))[:] + for i, c := range name { + if c == 0 { + name = name[:i] + break + } + } + // Check for useless names before allocating a string. + if string(name) == "." || string(name) == ".." { + continue + } + nm := string(name) + if err := fn(dirName, nm, newUnixDirent(dirName, nm, typ)); err != nil { + if err != ErrSkipFiles { + return err + } + skipFiles = true + } + } + + return nil +} + +const direntBufSize = 32 * 1024 + +var direntBufPool = sync.Pool{ + New: func() interface{} { + b := make([]byte, direntBufSize) + return &b + }, +} + +func readDir_Getdirentries(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + // This will cause the rest of the function to be omitted. + if !useGetdirentries { + panic("NOT IMPLEMENTED") + } + + fd, err := syscall.Open(dirName, syscall.O_RDONLY, 0) + if err != nil { + return &os.PathError{Op: "open", Path: dirName, Err: err} + } + defer syscall.Close(fd) + + p := direntBufPool.Get().(*[]byte) + defer direntBufPool.Put(p) + dbuf := *p + + var skipFiles bool + var basep uintptr + for { + length, err := getdirentries(fd, dbuf, &basep) + if err != nil { + return &os.PathError{Op: "getdirentries64", Path: dirName, Err: err} + } + if length == 0 { + break + } + buf := (*[1 << 30]byte)(unsafe.Pointer(&dbuf[0]))[:length:length] + for len(buf) > 0 { + dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0])) + buf = buf[dirent.Reclen:] + if dirent.Ino == 0 { + continue + } + typ := dtToType(dirent.Type) + if skipFiles && typ.IsRegular() { + continue + } + name := (*[len(syscall.Dirent{}.Name)]byte)(unsafe.Pointer(&dirent.Name))[:] + for i, c := range name { + if c == 0 { + name = name[:i] + break + } + } + if string(name) == "." || string(name) == ".." { + continue + } + nm := string(name) + if err := fn(dirName, nm, newUnixDirent(dirName, nm, typ)); err != nil { + if err != ErrSkipFiles { + return err + } + skipFiles = true + } + } + } + + return nil +} + +func dtToType(typ uint8) os.FileMode { + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) +} + +// Copied from syscall/syscall_unix.go + +// Do the interface allocations only once for common +// Errno values. +var ( + errEAGAIN error = syscall.EAGAIN + errEINVAL error = syscall.EINVAL + errENOENT error = syscall.ENOENT +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case syscall.EAGAIN: + return errEAGAIN + case syscall.EINVAL: + return errEINVAL + case syscall.ENOENT: + return errENOENT + } + return e +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_portable.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_portable.go new file mode 100644 index 0000000..bdd6050 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_portable.go @@ -0,0 +1,42 @@ +//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd) +// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd + +package fastwalk + +import ( + "io/fs" + "os" +) + +// readDir calls fn for each directory entry in dirName. +// It does not descend into directories or follow symlinks. +// If fn returns a non-nil error, readDir returns with that error +// immediately. +func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + f, err := os.Open(dirName) + if err != nil { + return err + } + des, readErr := f.ReadDir(-1) + f.Close() + if readErr != nil && len(des) == 0 { + return readErr + } + + var skipFiles bool + for _, d := range des { + if skipFiles && d.Type().IsRegular() { + continue + } + // Need to use FileMode.Type().Type() for fs.DirEntry + e := newDirEntry(dirName, d) + if err := fn(dirName, d.Name(), e); err != nil { + if err != ErrSkipFiles { + return err + } + skipFiles = true + } + } + + return readErr +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_unix.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix.go new file mode 100644 index 0000000..a7bfb1d --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix.go @@ -0,0 +1,155 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (linux || freebsd || openbsd || netbsd) && !appengine +// +build linux freebsd openbsd netbsd +// +build !appengine + +package fastwalk + +import ( + "fmt" + "io/fs" + "os" + "syscall" + "unsafe" +) + +const blockSize = 8 << 10 + +// unknownFileMode is a sentinel (and bogus) os.FileMode +// value used to represent a syscall.DT_UNKNOWN Dirent.Type. +const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice + +func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + fd, err := open(dirName, 0, 0) + if err != nil { + return &os.PathError{Op: "open", Path: dirName, Err: err} + } + defer syscall.Close(fd) + + // The buffer must be at least a block long. + buf := make([]byte, blockSize) // stack-allocated; doesn't escape + bufp := 0 // starting read position in buf + nbuf := 0 // end valid data in buf + skipFiles := false + for { + if bufp >= nbuf { + bufp = 0 + nbuf, err = readDirent(fd, buf) + if err != nil { + return os.NewSyscallError("readdirent", err) + } + if nbuf <= 0 { + return nil + } + } + consumed, name, typ := parseDirEnt(buf[bufp:nbuf]) + bufp += consumed + if name == "" || name == "." || name == ".." { + continue + } + // Fallback for filesystems (like old XFS) that don't + // support Dirent.Type and have DT_UNKNOWN (0) there + // instead. + if typ == unknownFileMode { + fi, err := os.Lstat(dirName + "/" + name) + if err != nil { + // It got deleted in the meantime. + if os.IsNotExist(err) { + continue + } + return err + } + typ = fi.Mode() & os.ModeType + } + if skipFiles && typ.IsRegular() { + continue + } + de := newUnixDirent(dirName, name, typ) + if err := fn(dirName, name, de); err != nil { + if err == ErrSkipFiles { + skipFiles = true + continue + } + return err + } + } +} + +func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode) { + // golang.org/issue/37269 + dirent := &syscall.Dirent{} + copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(dirent))[:], buf) + if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v { + panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v)) + } + if len(buf) < int(dirent.Reclen) { + panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen)) + } + consumed = int(dirent.Reclen) + if direntInode(dirent) == 0 { // File absent in directory. + return + } + switch dirent.Type { + case syscall.DT_REG: + typ = 0 + case syscall.DT_DIR: + typ = os.ModeDir + case syscall.DT_LNK: + typ = os.ModeSymlink + case syscall.DT_BLK: + typ = os.ModeDevice + case syscall.DT_FIFO: + typ = os.ModeNamedPipe + case syscall.DT_SOCK: + typ = os.ModeSocket + case syscall.DT_UNKNOWN: + typ = unknownFileMode + default: + // Skip weird things. + // It's probably a DT_WHT (http://lwn.net/Articles/325369/) + // or something. Revisit if/when this package is moved outside + // of goimports. goimports only cares about regular files, + // symlinks, and directories. + return + } + + nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) + nameLen := direntNamlen(dirent) + + // Special cases for common things: + if nameLen == 1 && nameBuf[0] == '.' { + name = "." + } else if nameLen == 2 && nameBuf[0] == '.' && nameBuf[1] == '.' { + name = ".." + } else { + name = string(nameBuf[:nameLen]) + } + return +} + +// According to https://golang.org/doc/go1.14#runtime +// A consequence of the implementation of preemption is that on Unix systems, including Linux and macOS +// systems, programs built with Go 1.14 will receive more signals than programs built with earlier releases. +// +// This causes syscall.Open and syscall.ReadDirent sometimes fail with EINTR errors. +// We need to retry in this case. +func open(path string, mode int, perm uint32) (fd int, err error) { + for { + fd, err := syscall.Open(path, mode, perm) + if err != syscall.EINTR { + return fd, err + } + } +} + +func readDirent(fd int, buf []byte) (n int, err error) { + for { + nbuf, err := syscall.ReadDirent(fd, buf) + if err != syscall.EINTR { + return nbuf, err + } + } +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_unix_bsd.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix_bsd.go new file mode 100644 index 0000000..db7f996 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix_bsd.go @@ -0,0 +1,18 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build freebsd || openbsd || netbsd +// +build freebsd openbsd netbsd + +package fastwalk + +import "syscall" + +func direntNamlen(dirent *syscall.Dirent) uint64 { + return uint64(dirent.Namlen) +} + +func direntInode(dirent *syscall.Dirent) uint64 { + return uint64(dirent.Fileno) +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_unix_linux.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix_linux.go new file mode 100644 index 0000000..a202f0c --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix_linux.go @@ -0,0 +1,33 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && !appengine +// +build linux,!appengine + +package fastwalk + +import ( + "syscall" + "unsafe" +) + +func direntNamlen(dirent *syscall.Dirent) uint64 { + const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name)) + nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) + const nameBufLen = uint16(len(nameBuf)) + limit := dirent.Reclen - fixedHdr + if limit > nameBufLen { + limit = nameBufLen + } + for i := uint64(0); i < uint64(limit); i++ { + if nameBuf[i] == 0 { + return i + } + } + panic("failed to find terminating 0 byte in dirent") +} + +func direntInode(dirent *syscall.Dirent) uint64 { + return uint64(dirent.Ino) +} diff --git a/vendor/github.com/charlievieth/fastwalk/go.LICENSE b/vendor/github.com/charlievieth/fastwalk/go.LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/go.LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin.go b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin.go new file mode 100644 index 0000000..19d955c --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin.go @@ -0,0 +1,56 @@ +//go:build darwin && go1.12 +// +build darwin,go1.12 + +package fastwalk + +import ( + "syscall" + "unsafe" +) + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) +func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:linkname syscall_syscall syscall.syscall +//go:linkname syscall_syscallPtr syscall.syscallPtr + +func closedir(dir uintptr) (err error) { + _, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, uintptr(dir), 0, 0) + if e1 != 0 { + err = e1 + } + return +} + +var libc_closedir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib" + +func readdir_r(dir uintptr, entry *syscall.Dirent, result **syscall.Dirent) syscall.Errno { + res, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, uintptr(dir), uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result))) + return syscall.Errno(res) +} + +var libc_readdir_r_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib" + +func opendir(path string) (dir uintptr, err error) { + // We implent opendir so that we don't have to open a file, duplicate + // it's FD, then call fdopendir with it. + p, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + r0, _, e1 := syscall_syscallPtr(libc_opendir_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0) + dir = uintptr(r0) + if e1 != 0 { + err = e1 + } + return dir, err +} + +var libc_opendir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_opendir opendir "/usr/lib/libSystem.B.dylib" diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s new file mode 100644 index 0000000..ccf2ff6 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s @@ -0,0 +1,28 @@ +//go:build go1.13 +// +build go1.13 + +#include "textflag.h" + +TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_closedir(SB) + +GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB) + +TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readdir_r(SB) + +GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB) + +TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_opendir(SB) + +GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB) + +TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc___getdirentries64(SB) + +GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8 +DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB) diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s new file mode 100644 index 0000000..ccf2ff6 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s @@ -0,0 +1,28 @@ +//go:build go1.13 +// +build go1.13 + +#include "textflag.h" + +TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_closedir(SB) + +GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB) + +TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readdir_r(SB) + +GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB) + +TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_opendir(SB) + +GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB) + +TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc___getdirentries64(SB) + +GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8 +DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB) diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go b/vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go new file mode 100644 index 0000000..e98c35f --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go @@ -0,0 +1,42 @@ +//go:build !nogetdirentries && darwin && go1.12 +// +build !nogetdirentries,darwin,go1.12 + +package fastwalk + +import ( + "syscall" + "unsafe" +) + +const useGetdirentries = true + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:linkname syscall_syscall6 syscall.syscall6 + +// Single-word zero for use when we need a valid pointer to 0 bytes. +var _zero uintptr + +func getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) { + var _p0 unsafe.Pointer + if len(buf) > 0 { + _p0 = unsafe.Pointer(&buf[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc___getdirentries64_trampoline_addr, + uintptr(fd), uintptr(_p0), uintptr(len(buf)), uintptr(unsafe.Pointer(basep)), + 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } else if n < 0 { + err = errnoErr(syscall.EINVAL) + } + return +} + +var libc___getdirentries64_trampoline_addr uintptr + +//go:cgo_import_dynamic libc___getdirentries64 __getdirentries64 "/usr/lib/libSystem.B.dylib" diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_no_getdirentries_darwin.go b/vendor/github.com/charlievieth/fastwalk/zsyscall_no_getdirentries_darwin.go new file mode 100644 index 0000000..3de3431 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_no_getdirentries_darwin.go @@ -0,0 +1,10 @@ +//go:build nogetdirentries && darwin && go1.12 +// +build nogetdirentries,darwin,go1.12 + +package fastwalk + +const useGetdirentries = false + +func getdirentries(fd int, _ []byte, _ *uintptr) (int, error) { + panic("NOT IMPLEMENTED") +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0f2940f..99520f1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,6 +4,9 @@ github.com/alecthomas/chroma/v2 github.com/alecthomas/chroma/v2/formatters/html github.com/alecthomas/chroma/v2/lexers github.com/alecthomas/chroma/v2/styles +# github.com/charlievieth/fastwalk v1.0.1 +## explicit; go 1.18 +github.com/charlievieth/fastwalk # github.com/dlclark/regexp2 v1.10.0 ## explicit; go 1.13 github.com/dlclark/regexp2