Replace filepath.WalkDir() with fastwalk.Walk()
This commit is contained in:
Seednode
parent
09ccf91a39
commit
194847fba0
10
cmd/files.go
10
cmd/files.go
|
|
@ -18,6 +18,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/charlievieth/fastwalk"
|
||||
"seedno.de/seednode/roulette/types"
|
||||
)
|
||||
|
||||
|
|
@ -227,8 +228,7 @@ func hasSupportedFiles(path string, formats *types.Types) (bool, error) {
|
|||
}
|
||||
|
||||
func pathCount(path string) (int, int, error) {
|
||||
var directories = 0
|
||||
var files = 0
|
||||
var directories, files = 0, 0
|
||||
|
||||
nodes, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
|
|
@ -243,7 +243,7 @@ func pathCount(path string) (int, int, error) {
|
|||
}
|
||||
}
|
||||
|
||||
return files, directories, nil
|
||||
return directories, files, nil
|
||||
}
|
||||
|
||||
func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats *scanStatsChannels, formats *types.Types) error {
|
||||
|
|
@ -252,7 +252,7 @@ func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats
|
|||
errorChannel := make(chan error)
|
||||
done := make(chan bool, 1)
|
||||
|
||||
filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error {
|
||||
fastwalk.Walk(nil, path, func(p string, info os.DirEntry, err error) error {
|
||||
switch {
|
||||
case !Recursive && info.IsDir() && p != path:
|
||||
return filepath.SkipDir
|
||||
|
|
@ -284,7 +284,7 @@ func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats
|
|||
stats.filesMatched <- 1
|
||||
}()
|
||||
case info.IsDir():
|
||||
files, directories, err := pathCount(p)
|
||||
directories, files, err := pathCount(p)
|
||||
if err != nil {
|
||||
errorChannel <- err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
ReleaseVersion string = "2.0.1"
|
||||
ReleaseVersion string = "2.1.0"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -95,7 +95,7 @@ func init() {
|
|||
rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)")
|
||||
rootCmd.Flags().BoolVar(&Handlers, "handlers", false, "display registered handlers (for debugging)")
|
||||
rootCmd.Flags().BoolVar(&Images, "images", false, "enable support for image files")
|
||||
rootCmd.Flags().BoolVarP(&Index, "index", "c", false, "generate index of supported file paths at startup")
|
||||
rootCmd.Flags().BoolVar(&Index, "index", false, "generate index of supported file paths at startup")
|
||||
rootCmd.Flags().StringVar(&IndexFile, "index-file", "", "path to optional persistent index file")
|
||||
rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints")
|
||||
rootCmd.Flags().IntVar(&MaxDirScans, "max-directory-scans", 32, "number of directories to scan at once")
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -4,6 +4,7 @@ go 1.21
|
|||
|
||||
require (
|
||||
github.com/alecthomas/chroma/v2 v2.9.1
|
||||
github.com/charlievieth/fastwalk v1.0.1
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/klauspost/compress v1.17.0
|
||||
github.com/spf13/cobra v1.7.0
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -4,6 +4,8 @@ github.com/alecthomas/chroma/v2 v2.9.1 h1:0O3lTQh9FxazJ4BYE/MOi/vDGuHn7B+6Bu902N
|
|||
github.com/alecthomas/chroma/v2 v2.9.1/go.mod h1:4TQu7gdfuPjSh76j78ietmqh9LiurGF0EpseFXdKMBw=
|
||||
github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
|
||||
github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/charlievieth/fastwalk v1.0.1 h1:jW01w8OCFdKS9JvAcnI+JHhWU/FuIEmNb24Ri9p7OVg=
|
||||
github.com/charlievieth/fastwalk v1.0.1/go.mod h1:dryXgMJyGHbMrAmmnF0/EJNBbZaihlwcNud5IuGyogU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
|
||||
github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
|
|
@ -13,6 +15,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
|||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
|
||||
github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
|
||||
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
|
||||
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
/vendor
|
||||
*.test
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2022 Charlie Vieth
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
.PHONY: test_build_darwin_arm64
|
||||
test_build_darwin_arm64:
|
||||
GOOS=darwin GOARCH=arm64 go test -c -o /dev/null
|
||||
|
||||
.PHONY: test_build_darwin_amd64
|
||||
test_build_darwin_amd64:
|
||||
GOOS=darwin GOARCH=amd64 go test -c -o /dev/null
|
||||
|
||||
.PHONY: test_build_linux_arm64
|
||||
test_build_linux_arm64:
|
||||
GOOS=linux GOARCH=arm64 go test -c -o /dev/null
|
||||
|
||||
.PHONY: test_build_linux_amd64
|
||||
test_build_linux_amd64:
|
||||
GOOS=linux GOARCH=amd64 go test -c -o /dev/null
|
||||
|
||||
.PHONY: test_build_windows_amd64
|
||||
test_build_windows_amd64:
|
||||
GOOS=windows GOARCH=amd64 go test -c -o /dev/null
|
||||
|
||||
.PHONY: test_build_freebsd_amd64
|
||||
test_build_freebsd_amd64:
|
||||
GOOS=freebsd GOARCH=amd64 go test -c -o /dev/null
|
||||
|
||||
.PHONY: test_build_openbsd_amd64
|
||||
test_build_openbsd_amd64:
|
||||
GOOS=openbsd GOARCH=amd64 go test -c -o /dev/null
|
||||
|
||||
.PHONY: test_build_netbsd_amd64
|
||||
test_build_netbsd_amd64:
|
||||
GOOS=netbsd GOARCH=amd64 go test -c -o /dev/null
|
||||
|
||||
# Test that we can build fastwalk on multiple platforms
|
||||
.PHONY: test_build
|
||||
test_build: test_build_darwin_arm64 test_build_darwin_amd64 \
|
||||
test_build_linux_arm64 test_build_linux_amd64 \
|
||||
test_build_windows_amd64 test_build_freebsd_amd64 \
|
||||
test_build_openbsd_amd64 test_build_netbsd_amd64
|
||||
|
||||
.PHONY: test
|
||||
test: # runs all tests against the package with race detection and coverage percentage
|
||||
@go test -race -cover ./...
|
||||
ifeq "$(shell go env GOOS)" "darwin"
|
||||
@go test -tags nogetdirentries -race -cover ./...
|
||||
endif
|
||||
|
||||
.PHONY: quick
|
||||
quick: # runs all tests without coverage or the race detector
|
||||
@go test ./...
|
||||
|
||||
.PHONY: bench
|
||||
bench:
|
||||
@go test -run '^$' -bench . -benchmem ./...
|
||||
|
||||
.PHONY: bench_comp
|
||||
bench_comp:
|
||||
@go run ./scripts/bench_comp.go
|
||||
|
||||
.PHONY: all
|
||||
all: test test_build
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
[](https://pkg.go.dev/github.com/charlievieth/fastwalk)
|
||||
[](https://github.com/charlievieth/fastwalk/actions/workflows/macos.yml)
|
||||
[](https://github.com/charlievieth/fastwalk/actions/workflows/linux.yml)
|
||||
[](https://github.com/charlievieth/fastwalk/actions/workflows/windows.yml)
|
||||
|
||||
# fastwalk
|
||||
|
||||
Fast parallel directory traversal for Golang.
|
||||
|
||||
Package fastwalk provides a fast parallel version of [`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc)
|
||||
that is \~2x faster on macOS, \~4x faster on Linux, \~6x faster on Windows,
|
||||
allocates 50% less memory, and requires 25% fewer memory allocations.
|
||||
Additionally, it is \~4-5x faster than [godirwalk](https://github.com/karrick/godirwalk)
|
||||
across OSes.
|
||||
|
||||
Inspired by and based off of [golang.org/x/tools/internal/fastwalk](https://pkg.go.dev/golang.org/x/tools@v0.1.9/internal/fastwalk).
|
||||
|
||||
## Features
|
||||
|
||||
* Fast: multiple goroutines stat the filesystem and call the
|
||||
[`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc) callback concurrently
|
||||
* Safe symbolic link traversal ([`Config.Follow`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Config))
|
||||
* Same behavior and callback signature as [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir)
|
||||
* Wrapper functions are provided to ignore duplicate files and directories:
|
||||
[`IgnoreDuplicateFiles()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#IgnoreDuplicateFiles)
|
||||
and
|
||||
[`IgnoreDuplicateDirs()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#IgnoreDuplicateDirs)
|
||||
* Extensively tested on macOS, Linux, and Windows
|
||||
|
||||
## Usage
|
||||
|
||||
Usage is the same as [`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc),
|
||||
but the [`walkFn`](https://pkg.go.dev/path/filepath@go1.17.7#WalkFunc)
|
||||
argument to [`fastwalk.Walk`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk)
|
||||
must be safe for concurrent use.
|
||||
|
||||
Examples can be found in the [examples](./examples) directory.
|
||||
|
||||
<!-- TODO: this example is large move it to an examples folder -->
|
||||
|
||||
The below example is a very simple version of the POSIX
|
||||
[find](https://pubs.opengroup.org/onlinepubs/007904975/utilities/find.html) utility:
|
||||
```go
|
||||
// fwfind is a an example program that is similar to POSIX find,
|
||||
// but faster and worse (it's an example).
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/charlievieth/fastwalk"
|
||||
)
|
||||
|
||||
const usageMsg = `Usage: %[1]s [-L] [-name] [PATH...]:
|
||||
|
||||
%[1]s is a poor replacement for the POSIX find utility
|
||||
|
||||
`
|
||||
|
||||
func main() {
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stdout, usageMsg, filepath.Base(os.Args[0]))
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
pattern := flag.String("name", "", "Pattern to match file names against.")
|
||||
followLinks := flag.Bool("L", false, "Follow symbolic links")
|
||||
flag.Parse()
|
||||
|
||||
// If no paths are provided default to the current directory: "."
|
||||
args := flag.Args()
|
||||
if len(args) == 0 {
|
||||
args = append(args, ".")
|
||||
}
|
||||
|
||||
// Follow links if the "-L" flag is provided
|
||||
conf := fastwalk.Config{
|
||||
Follow: *followLinks,
|
||||
}
|
||||
|
||||
walkFn := func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
|
||||
return nil // returning the error stops iteration
|
||||
}
|
||||
if *pattern != "" {
|
||||
if ok, err := filepath.Match(*pattern, d.Name()); !ok {
|
||||
// invalid pattern (err != nil) or name does not match
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = fmt.Println(path)
|
||||
return err
|
||||
}
|
||||
for _, root := range args {
|
||||
if err := fastwalk.Walk(&conf, root, walkFn); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", root, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Benchmarks were created using `go1.17.6` and can be generated with the `bench_comp` make target:
|
||||
```sh
|
||||
$ make bench_comp
|
||||
```
|
||||
|
||||
### Darwin
|
||||
|
||||
**Hardware:**
|
||||
```
|
||||
goos: darwin
|
||||
goarch: arm64
|
||||
cpu: Apple M1 Max
|
||||
```
|
||||
|
||||
#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk):
|
||||
```
|
||||
filepath fastwalk delta
|
||||
time/op 27.9ms ± 1% 13.0ms ± 1% -53.33%
|
||||
alloc/op 4.33MB ± 0% 2.14MB ± 0% -50.55%
|
||||
allocs/op 50.9k ± 0% 37.7k ± 0% -26.01%
|
||||
```
|
||||
|
||||
#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk):
|
||||
```
|
||||
godirwalk fastwalk delta
|
||||
time/op 58.5ms ± 3% 18.0ms ± 2% -69.30%
|
||||
alloc/op 25.3MB ± 0% 2.1MB ± 0% -91.55%
|
||||
allocs/op 57.6k ± 0% 37.7k ± 0% -34.59%
|
||||
```
|
||||
|
||||
### Linux
|
||||
|
||||
**Hardware:**
|
||||
```
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
cpu: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz
|
||||
drive: Samsung SSD 970 PRO 1TB
|
||||
```
|
||||
|
||||
#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk):
|
||||
|
||||
```
|
||||
filepath fastwalk delta
|
||||
time/op 10.1ms ± 2% 2.8ms ± 2% -72.83%
|
||||
alloc/op 2.44MB ± 0% 1.70MB ± 0% -30.46%
|
||||
allocs/op 47.2k ± 0% 36.9k ± 0% -21.80%
|
||||
```
|
||||
|
||||
#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk):
|
||||
|
||||
```
|
||||
filepath fastwalk delta
|
||||
time/op 13.7ms ±16% 2.8ms ± 2% -79.88%
|
||||
alloc/op 7.48MB ± 0% 1.70MB ± 0% -77.34%
|
||||
allocs/op 53.8k ± 0% 36.9k ± 0% -31.38%
|
||||
```
|
||||
|
||||
### Windows
|
||||
|
||||
**Hardware:**
|
||||
```
|
||||
goos: windows
|
||||
goarch: amd64
|
||||
pkg: github.com/charlievieth/fastwalk
|
||||
cpu: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz
|
||||
```
|
||||
|
||||
#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk):
|
||||
|
||||
```
|
||||
filepath fastwalk delta
|
||||
time/op 88.0ms ± 1% 14.6ms ± 1% -83.47%
|
||||
alloc/op 5.68MB ± 0% 6.76MB ± 0% +19.01%
|
||||
allocs/op 69.6k ± 0% 90.4k ± 0% +29.87%
|
||||
```
|
||||
|
||||
#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk):
|
||||
|
||||
```
|
||||
filepath fastwalk delta
|
||||
time/op 87.4ms ± 1% 14.6ms ± 1% -83.34%
|
||||
alloc/op 6.14MB ± 0% 6.76MB ± 0% +10.24%
|
||||
allocs/op 100k ± 0% 90k ± 0% -9.59%
|
||||
```
|
||||
|
||||
## Darwin: getdirentries64
|
||||
|
||||
The `nogetdirentries` build tag can be used to prevent `fastwalk` from using
|
||||
and linking to the non-public `__getdirentries64` syscall. This is required
|
||||
if an app using `fastwalk` is to be distributed via Apple's App Store (see
|
||||
https://github.com/golang/go/issues/30933 for more details). When using
|
||||
`__getdirentries64` is disabled, `fastwalk` will use `readdir_r` instead,
|
||||
which is what the Go standard library uses for
|
||||
[`os.ReadDir`](https://pkg.go.dev/os#ReadDir) and is about \~10% slower than
|
||||
`__getdirentries64`
|
||||
([benchmarks](https://github.com/charlievieth/fastwalk/blob/2e6a1b8a1ce88e578279e6e631b2129f7144ec87/fastwalk_darwin_test.go#L19-L57)).
|
||||
|
||||
Example of how to build and test that your program is not linked to `__getdirentries64`:
|
||||
```sh
|
||||
# NOTE: the following only applies to darwin (aka macOS)
|
||||
|
||||
# Build binary that imports fastwalk without linking to __getdirentries64.
|
||||
$ go build -tags nogetdirentries -o YOUR_BINARY
|
||||
# Test that __getdirentries64 is not linked (this should print no output).
|
||||
$ ! otool -dyld_info YOUR_BINARY | grep -F getdirentries64
|
||||
```
|
||||
|
||||
There is a also a script [scripts/links2getdirentries.bash](scripts/links2getdirentries.bash)
|
||||
that can be used to check if a program binary links to getdirentries.
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func isDir(path string, d fs.DirEntry) bool {
|
||||
if d.IsDir() {
|
||||
return true
|
||||
}
|
||||
if d.Type()&os.ModeSymlink != 0 {
|
||||
if fi, err := StatDirEntry(path, d); err == nil {
|
||||
return fi.IsDir()
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IgnoreDuplicateDirs wraps fs.WalkDirFunc walkFn to make it follow symbolic
|
||||
// links and ignore duplicate directories (if a symlink points to a directory
|
||||
// that has already been traversed it is skipped). The walkFn is called for
|
||||
// for skipped directories, but the directory is not traversed (this is
|
||||
// required for error handling).
|
||||
//
|
||||
// The Config.Follow setting has no effect on the behavior of Walk when
|
||||
// this wrapper is used.
|
||||
//
|
||||
// In most use cases, the returned fs.WalkDirFunc should not be reused between
|
||||
// in another call to Walk. If it is reused, any previously visited file will
|
||||
// be skipped.
|
||||
//
|
||||
// NOTE: The order of traversal is undefined. Given an "example" directory
|
||||
// like the one below where "dir" is a directory and "smydir1" and "smydir2"
|
||||
// are links to it, only one of "dir", "smydir1", or "smydir2" will be
|
||||
// traversed, but which one is undefined.
|
||||
//
|
||||
// example
|
||||
// ├── dir
|
||||
// ├── smydir1 -> dir
|
||||
// └── smydir2 -> dir
|
||||
//
|
||||
func IgnoreDuplicateDirs(walkFn fs.WalkDirFunc) fs.WalkDirFunc {
|
||||
filter := NewEntryFilter()
|
||||
return func(path string, d fs.DirEntry, err error) error {
|
||||
// Call walkFn before checking the entry filter so that we
|
||||
// don't record directories that are skipped with SkipDir.
|
||||
err = walkFn(path, d, err)
|
||||
if err != nil {
|
||||
if err != filepath.SkipDir && isDir(path, d) {
|
||||
filter.Entry(path, d)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if isDir(path, d) {
|
||||
if filter.Entry(path, d) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if d.Type() == os.ModeSymlink {
|
||||
return ErrTraverseLink
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// IgnoreDuplicateFiles wraps walkFn so that symlinks are followed and duplicate
|
||||
// files are ignored. If a symlink resolves to a file that has already been
|
||||
// visited it will be skipped.
|
||||
//
|
||||
// In most use cases, the returned fs.WalkDirFunc should not be reused between
|
||||
// in another call to Walk. If it is reused, any previously visited file will
|
||||
// be skipped.
|
||||
//
|
||||
// This can significantly slow Walk as os.Stat() is called for each path
|
||||
// (on Windows, os.Stat() is only needed for symlinks).
|
||||
func IgnoreDuplicateFiles(walkFn fs.WalkDirFunc) fs.WalkDirFunc {
|
||||
filter := NewEntryFilter()
|
||||
return func(path string, d fs.DirEntry, err error) error {
|
||||
// Skip all duplicate files, directories, and links
|
||||
if filter.Entry(path, d) {
|
||||
if isDir(path, d) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err = walkFn(path, d, err)
|
||||
if err == nil && d.Type() == os.ModeSymlink && isDir(path, d) {
|
||||
err = ErrTraverseLink
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// IgnorePermissionErrors wraps walkFn so that permission errors are ignored.
|
||||
// The returned fs.WalkDirFunc may be reused.
|
||||
func IgnorePermissionErrors(walkFn fs.WalkDirFunc) fs.WalkDirFunc {
|
||||
return func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil && os.IsPermission(err) {
|
||||
return nil
|
||||
}
|
||||
return walkFn(path, d, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
type fileInfo struct {
|
||||
once sync.Once
|
||||
fs.FileInfo
|
||||
err error
|
||||
}
|
||||
|
||||
func loadFileInfo(pinfo **fileInfo) *fileInfo {
|
||||
ptr := (*unsafe.Pointer)(unsafe.Pointer(pinfo))
|
||||
fi := (*fileInfo)(atomic.LoadPointer(ptr))
|
||||
if fi == nil {
|
||||
fi = &fileInfo{}
|
||||
if !atomic.CompareAndSwapPointer(
|
||||
(*unsafe.Pointer)(unsafe.Pointer(pinfo)), // adrr
|
||||
nil, // old
|
||||
unsafe.Pointer(fi), // new
|
||||
) {
|
||||
fi = (*fileInfo)(atomic.LoadPointer(ptr))
|
||||
}
|
||||
}
|
||||
return fi
|
||||
}
|
||||
|
||||
// StatDirEntry returns the fs.FileInfo for the file or subdirectory described
|
||||
// by the entry. If the entry is a symbolic link, StatDirEntry returns the
|
||||
// fs.FileInfo for the file the line references (os.Stat).
|
||||
// If fs.DirEntry de is a fastwalk.DirEntry it's Stat() method is used and the
|
||||
// returned fs.FileInfo may be a previously cached result.
|
||||
func StatDirEntry(path string, de fs.DirEntry) (fs.FileInfo, error) {
|
||||
if de.Type()&os.ModeSymlink == 0 {
|
||||
return de.Info()
|
||||
}
|
||||
if d, ok := de.(DirEntry); ok {
|
||||
return d.Stat()
|
||||
}
|
||||
return os.Stat(path)
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd)
|
||||
// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
)
|
||||
|
||||
type portableDirent struct {
|
||||
fs.DirEntry
|
||||
path string
|
||||
stat *fileInfo
|
||||
}
|
||||
|
||||
// TODO: cache the result of Stat
|
||||
func (d *portableDirent) Stat() (fs.FileInfo, error) {
|
||||
if d.DirEntry.Type()&os.ModeSymlink == 0 {
|
||||
return d.DirEntry.Info()
|
||||
}
|
||||
stat := loadFileInfo(&d.stat)
|
||||
stat.once.Do(func() {
|
||||
stat.FileInfo, stat.err = os.Stat(d.path)
|
||||
})
|
||||
return stat.FileInfo, stat.err
|
||||
}
|
||||
|
||||
func newDirEntry(dirName string, info fs.DirEntry) fs.DirEntry {
|
||||
return &portableDirent{
|
||||
DirEntry: info,
|
||||
path: dirName + string(os.PathSeparator) + info.Name(),
|
||||
}
|
||||
}
|
||||
|
||||
func fileInfoToDirEntry(dirname string, fi fs.FileInfo) fs.DirEntry {
|
||||
return newDirEntry(dirname, fs.FileInfoToDirEntry(fi))
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
//go:build (aix || darwin || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd || solaris) && !appengine
|
||||
// +build aix darwin dragonfly freebsd js,wasm linux netbsd openbsd solaris
|
||||
// +build !appengine
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
)
|
||||
|
||||
type unixDirent struct {
|
||||
parent string
|
||||
name string
|
||||
typ os.FileMode
|
||||
info *fileInfo
|
||||
stat *fileInfo
|
||||
}
|
||||
|
||||
func (d *unixDirent) Name() string { return d.name }
|
||||
func (d *unixDirent) IsDir() bool { return d.typ.IsDir() }
|
||||
func (d *unixDirent) Type() os.FileMode { return d.typ }
|
||||
|
||||
func (d *unixDirent) Info() (fs.FileInfo, error) {
|
||||
info := loadFileInfo(&d.info)
|
||||
info.once.Do(func() {
|
||||
info.FileInfo, info.err = os.Lstat(d.parent + "/" + d.name)
|
||||
})
|
||||
return info.FileInfo, info.err
|
||||
}
|
||||
|
||||
func (d *unixDirent) Stat() (fs.FileInfo, error) {
|
||||
if d.typ&os.ModeSymlink == 0 {
|
||||
return d.Info()
|
||||
}
|
||||
stat := loadFileInfo(&d.stat)
|
||||
stat.once.Do(func() {
|
||||
stat.FileInfo, stat.err = os.Stat(d.parent + "/" + d.name)
|
||||
})
|
||||
return stat.FileInfo, stat.err
|
||||
}
|
||||
|
||||
func newUnixDirent(parent, name string, typ os.FileMode) *unixDirent {
|
||||
return &unixDirent{
|
||||
parent: parent,
|
||||
name: name,
|
||||
typ: typ,
|
||||
}
|
||||
}
|
||||
|
||||
func fileInfoToDirEntry(dirname string, fi fs.FileInfo) fs.DirEntry {
|
||||
info := &fileInfo{
|
||||
FileInfo: fi,
|
||||
}
|
||||
info.once.Do(func() {})
|
||||
return &unixDirent{
|
||||
parent: dirname,
|
||||
name: fi.Name(),
|
||||
typ: fi.Mode().Type(),
|
||||
info: info,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd && !windows)
|
||||
// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd,!windows
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type EntryFilter struct {
|
||||
// we assume most files have not been seen so
|
||||
// no need for a RWMutex
|
||||
mu sync.Mutex
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
func (e *EntryFilter) Entry(path string, _ fs.DirEntry) bool {
|
||||
name, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
e.mu.Lock()
|
||||
if e.seen == nil {
|
||||
e.seen = make(map[string]struct{}, 128)
|
||||
}
|
||||
_, ok := e.seen[name]
|
||||
if !ok {
|
||||
e.seen[name] = struct{}{}
|
||||
}
|
||||
e.mu.Unlock()
|
||||
return ok
|
||||
}
|
||||
|
||||
func NewEntryFilter() *EntryFilter {
|
||||
return &EntryFilter{seen: make(map[string]struct{}, 128)}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
//go:build (linux || darwin || freebsd || openbsd || netbsd || !windows) && !appengine
|
||||
// +build linux darwin freebsd openbsd netbsd !windows
|
||||
// +build !appengine
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"sync"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type fileKey struct {
|
||||
Dev uint64
|
||||
Ino uint64
|
||||
}
|
||||
|
||||
type entryMap struct {
|
||||
mu sync.Mutex
|
||||
keys map[fileKey]struct{}
|
||||
}
|
||||
|
||||
// An EntryFilter keeps track of visited directory entries and can be used to
|
||||
// detect and avoid symlink loops or processing the same file twice.
|
||||
type EntryFilter struct {
|
||||
// Use an array of 8 to reduce lock contention. The entryMap is
|
||||
// picked via the inode number. We don't take the device number
|
||||
// into account because: we don't expect to see many of them and
|
||||
// uniformly spreading the load isn't terribly beneficial here.
|
||||
ents [8]entryMap
|
||||
}
|
||||
|
||||
// NewEntryFilter returns a new EntryFilter
|
||||
func NewEntryFilter() *EntryFilter {
|
||||
return new(EntryFilter)
|
||||
}
|
||||
|
||||
func (e *EntryFilter) seen(dev, ino uint64) (seen bool) {
|
||||
m := &e.ents[ino%uint64(len(e.ents))]
|
||||
m.mu.Lock()
|
||||
if _, seen = m.keys[fileKey{dev, ino}]; !seen {
|
||||
if m.keys == nil {
|
||||
m.keys = make(map[fileKey]struct{})
|
||||
}
|
||||
m.keys[fileKey{dev, ino}] = struct{}{}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return seen
|
||||
}
|
||||
|
||||
// TODO: this name is confusing and should be fixed
|
||||
|
||||
// Entry returns if path and fs.DirEntry have been seen before.
|
||||
func (e *EntryFilter) Entry(path string, de fs.DirEntry) (seen bool) {
|
||||
fi, err := StatDirEntry(path, de)
|
||||
if err != nil {
|
||||
return true // treat errors as duplicate files
|
||||
}
|
||||
stat := fi.Sys().(*syscall.Stat_t)
|
||||
return e.seen(uint64(stat.Dev), uint64(stat.Ino))
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
//go:build windows && !appengine
|
||||
// +build windows,!appengine
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type fileKey struct {
|
||||
VolumeSerialNumber uint32
|
||||
FileIndexHigh uint32
|
||||
FileIndexLow uint32
|
||||
}
|
||||
|
||||
type EntryFilter struct {
|
||||
mu sync.Mutex
|
||||
seen map[fileKey]struct{}
|
||||
}
|
||||
|
||||
func NewEntryFilter() *EntryFilter {
|
||||
return &EntryFilter{seen: make(map[fileKey]struct{}, 128)}
|
||||
}
|
||||
|
||||
func (e *EntryFilter) Entry(path string, _ fs.DirEntry) bool {
|
||||
namep, err := syscall.UTF16PtrFromString(fixLongPath(path))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
h, err := syscall.CreateFile(namep, 0, 0, nil, syscall.OPEN_EXISTING,
|
||||
syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
var d syscall.ByHandleFileInformation
|
||||
err = syscall.GetFileInformationByHandle(h, &d)
|
||||
syscall.CloseHandle(h)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
key := fileKey{
|
||||
VolumeSerialNumber: d.VolumeSerialNumber,
|
||||
FileIndexHigh: d.FileIndexHigh,
|
||||
FileIndexLow: d.FileIndexLow,
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
if e.seen == nil {
|
||||
e.seen = make(map[fileKey]struct{})
|
||||
}
|
||||
_, ok := e.seen[key]
|
||||
if !ok {
|
||||
e.seen[key] = struct{}{}
|
||||
}
|
||||
e.mu.Unlock()
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
func isAbs(path string) (b bool) {
|
||||
v := filepath.VolumeName(path)
|
||||
if v == "" {
|
||||
return false
|
||||
}
|
||||
path = path[len(v):]
|
||||
if path == "" {
|
||||
return false
|
||||
}
|
||||
return os.IsPathSeparator(path[0])
|
||||
}
|
||||
|
||||
// fixLongPath returns the extended-length (\\?\-prefixed) form of
|
||||
// path when needed, in order to avoid the default 260 character file
|
||||
// path limit imposed by Windows. If path is not easily converted to
|
||||
// the extended-length form (for example, if path is a relative path
|
||||
// or contains .. elements), or is short enough, fixLongPath returns
|
||||
// path unmodified.
|
||||
//
|
||||
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
|
||||
func fixLongPath(path string) string {
|
||||
// Do nothing (and don't allocate) if the path is "short".
|
||||
// Empirically (at least on the Windows Server 2013 builder),
|
||||
// the kernel is arbitrarily okay with < 248 bytes. That
|
||||
// matches what the docs above say:
|
||||
// "When using an API to create a directory, the specified
|
||||
// path cannot be so long that you cannot append an 8.3 file
|
||||
// name (that is, the directory name cannot exceed MAX_PATH
|
||||
// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
|
||||
//
|
||||
// The MSDN docs appear to say that a normal path that is 248 bytes long
|
||||
// will work; empirically the path must be less then 248 bytes long.
|
||||
if len(path) < 248 {
|
||||
// Don't fix. (This is how Go 1.7 and earlier worked,
|
||||
// not automatically generating the \\?\ form)
|
||||
return path
|
||||
}
|
||||
|
||||
// The extended form begins with \\?\, as in
|
||||
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
|
||||
// The extended form disables evaluation of . and .. path
|
||||
// elements and disables the interpretation of / as equivalent
|
||||
// to \. The conversion here rewrites / to \ and elides
|
||||
// . elements as well as trailing or duplicate separators. For
|
||||
// simplicity it avoids the conversion entirely for relative
|
||||
// paths or paths containing .. elements. For now,
|
||||
// \\server\share paths are not converted to
|
||||
// \\?\UNC\server\share paths because the rules for doing so
|
||||
// are less well-specified.
|
||||
if len(path) >= 2 && path[:2] == `\\` {
|
||||
// Don't canonicalize UNC paths.
|
||||
return path
|
||||
}
|
||||
if !isAbs(path) {
|
||||
// Relative path
|
||||
return path
|
||||
}
|
||||
|
||||
const prefix = `\\?`
|
||||
|
||||
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
|
||||
copy(pathbuf, prefix)
|
||||
n := len(path)
|
||||
r, w := 0, len(prefix)
|
||||
for r < n {
|
||||
switch {
|
||||
case os.IsPathSeparator(path[r]):
|
||||
// empty block
|
||||
r++
|
||||
case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
|
||||
// /./
|
||||
r++
|
||||
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
|
||||
// /../ is currently unhandled
|
||||
return path
|
||||
default:
|
||||
pathbuf[w] = '\\'
|
||||
w++
|
||||
for ; r < n && !os.IsPathSeparator(path[r]); r++ {
|
||||
pathbuf[w] = path[r]
|
||||
w++
|
||||
}
|
||||
}
|
||||
}
|
||||
// A drive's root directory needs a trailing \
|
||||
if w == len(`\\?\c:`) {
|
||||
pathbuf[w] = '\\'
|
||||
w++
|
||||
}
|
||||
return string(pathbuf[:w])
|
||||
}
|
||||
|
|
@ -0,0 +1,394 @@
|
|||
// Package fastwalk provides a faster version of filepath.Walk for file system
|
||||
// scanning tools.
|
||||
package fastwalk
|
||||
|
||||
/*
|
||||
* This code borrows heavily from golang.org/x/tools/internal/fastwalk
|
||||
* and as such the Go license can be found in the go.LICENSE file and
|
||||
* is reproduced below:
|
||||
*
|
||||
* Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Google Inc. nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the
|
||||
// symlink named in the call may be traversed.
|
||||
var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory")
|
||||
|
||||
// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the
|
||||
// callback should not be called for any other files in the current directory.
|
||||
// Child directories will still be traversed.
|
||||
var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
|
||||
|
||||
// SkipDir is used as a return value from WalkDirFuncs to indicate that
|
||||
// the directory named in the call is to be skipped. It is not returned
|
||||
// as an error by any function.
|
||||
var SkipDir = fs.SkipDir
|
||||
|
||||
// DefaultNumWorkers returns the default number of worker goroutines to use in
|
||||
// fastwalk.Walk and is the value of runtime.GOMAXPROCS(-1) clamped to a range
|
||||
// of 4 to 32 except on Darwin where it is either 4 (8 cores or less) or 6
|
||||
// (more than 8 cores). This is because Walk / IO performance on Darwin
|
||||
// degrades with more concurrency.
|
||||
//
|
||||
// The optimal number for your workload may be lower or higher. The results
|
||||
// of BenchmarkFastWalkNumWorkers benchmark may be informative.
|
||||
func DefaultNumWorkers() int {
|
||||
numCPU := runtime.GOMAXPROCS(-1)
|
||||
if numCPU < 4 {
|
||||
return 4
|
||||
}
|
||||
// Darwin IO performance on APFS slows with more workers.
|
||||
// Stat performance is best around 2-4 and file IO is best
|
||||
// around 4-6. More workers only benefit CPU intensive tasks.
|
||||
if runtime.GOOS == "darwin" {
|
||||
if numCPU <= 8 {
|
||||
return 4
|
||||
}
|
||||
return 6
|
||||
}
|
||||
if numCPU > 32 {
|
||||
return 32
|
||||
}
|
||||
return numCPU
|
||||
}
|
||||
|
||||
// DefaultConfig is the default Config used when none is supplied.
|
||||
var DefaultConfig = Config{
|
||||
Follow: false,
|
||||
NumWorkers: DefaultNumWorkers(),
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
// TODO: do we want to pass a sentinel error to WalkFunc if
|
||||
// a symlink loop is detected?
|
||||
|
||||
// Follow symbolic links ignoring directories that would lead
|
||||
// to infinite loops; that is, entering a previously visited
|
||||
// directory that is an ancestor of the last file encountered.
|
||||
//
|
||||
// The sentinel error ErrTraverseLink is ignored when Follow
|
||||
// is true (this to prevent users from defeating the loop
|
||||
// detection logic), but SkipDir and ErrSkipFiles are still
|
||||
// respected.
|
||||
Follow bool
|
||||
|
||||
// Number of parallel workers to use. If NumWorkers if ≤ 0 then
|
||||
// the greater of runtime.NumCPU() or 4 is used.
|
||||
NumWorkers int
|
||||
}
|
||||
|
||||
// A DirEntry extends the fs.DirEntry interface to add a Stat() method
|
||||
// that returns the result of calling os.Stat() on the underlying file.
|
||||
// The results of Info() and Stat() are cached.
|
||||
//
|
||||
// The fs.DirEntry argument passed to the fs.WalkDirFunc by Walk is
|
||||
// always a DirEntry. The only exception is the root directory with
|
||||
// with Walk is called.
|
||||
type DirEntry interface {
|
||||
fs.DirEntry
|
||||
|
||||
// Stat returns the FileInfo for the file or subdirectory described
|
||||
// by the entry. The returned FileInfo may be from the time of the
|
||||
// original directory read or from the time of the call to Stat.
|
||||
// If the entry denotes a symbolic link, Stat reports the information
|
||||
// about the target itself, not the link.
|
||||
Stat() (fs.FileInfo, error)
|
||||
}
|
||||
|
||||
// Walk is a faster implementation of filepath.Walk.
|
||||
//
|
||||
// filepath.Walk's design necessarily calls os.Lstat on each file, even if
|
||||
// the caller needs less info. Many tools need only the type of each file.
|
||||
// On some platforms, this information is provided directly by the readdir
|
||||
// system call, avoiding the need to stat each file individually.
|
||||
// fastwalk_unix.go contains a fork of the syscall routines.
|
||||
//
|
||||
// See golang.org/issue/16399
|
||||
//
|
||||
// Walk walks the file tree rooted at root, calling walkFn for each file or
|
||||
// directory in the tree, including root.
|
||||
//
|
||||
// If walkFn returns filepath.SkipDir, the directory is skipped.
|
||||
//
|
||||
// Unlike filepath.WalkDir:
|
||||
// * File stat calls must be done by the user and should be done via
|
||||
// the DirEntry argument to walkFn since it caches the results of
|
||||
// Stat and Lstat.
|
||||
// * The fs.DirEntry argument is always a fastwalk.DirEntry, which has
|
||||
// a Stat() method that returns the result of calling os.Stat() on the
|
||||
// file. The result of Stat() may be cached.
|
||||
// * Multiple goroutines stat the filesystem concurrently. The provided
|
||||
// walkFn must be safe for concurrent use.
|
||||
// * Walk can follow symlinks if walkFn returns the ErrTraverseLink
|
||||
// sentinel error. It is the walkFn's responsibility to prevent
|
||||
// Walk from going into symlink cycles.
|
||||
func Walk(conf *Config, root string, walkFn fs.WalkDirFunc) error {
|
||||
if conf == nil {
|
||||
dupe := DefaultConfig
|
||||
conf = &dupe
|
||||
}
|
||||
fi, err := os.Lstat(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make sure to wait for all workers to finish, otherwise
|
||||
// walkFn could still be called after returning. This Wait call
|
||||
// runs after close(e.donec) below.
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
|
||||
numWorkers := conf.NumWorkers
|
||||
if numWorkers <= 0 {
|
||||
numWorkers = DefaultNumWorkers()
|
||||
}
|
||||
|
||||
w := &walker{
|
||||
fn: walkFn,
|
||||
enqueuec: make(chan walkItem, numWorkers), // buffered for performance
|
||||
workc: make(chan walkItem, numWorkers), // buffered for performance
|
||||
donec: make(chan struct{}),
|
||||
|
||||
// buffered for correctness & not leaking goroutines:
|
||||
resc: make(chan error, numWorkers),
|
||||
|
||||
follow: conf.Follow,
|
||||
}
|
||||
if w.follow {
|
||||
if fi, err := os.Stat(root); err == nil {
|
||||
w.ignoredDirs = append(w.ignoredDirs, fi)
|
||||
}
|
||||
}
|
||||
|
||||
defer close(w.donec)
|
||||
|
||||
for i := 0; i < numWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go w.doWork(&wg)
|
||||
}
|
||||
|
||||
root = cleanRootPath(root)
|
||||
todo := []walkItem{{dir: root, info: fileInfoToDirEntry(filepath.Dir(root), fi)}}
|
||||
out := 0
|
||||
for {
|
||||
workc := w.workc
|
||||
var workItem walkItem
|
||||
if len(todo) == 0 {
|
||||
workc = nil
|
||||
} else {
|
||||
workItem = todo[len(todo)-1]
|
||||
}
|
||||
select {
|
||||
case workc <- workItem:
|
||||
todo = todo[:len(todo)-1]
|
||||
out++
|
||||
case it := <-w.enqueuec:
|
||||
todo = append(todo, it)
|
||||
case err := <-w.resc:
|
||||
out--
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if out == 0 && len(todo) == 0 {
|
||||
// It's safe to quit here, as long as the buffered
|
||||
// enqueue channel isn't also readable, which might
|
||||
// happen if the worker sends both another unit of
|
||||
// work and its result before the other select was
|
||||
// scheduled and both w.resc and w.enqueuec were
|
||||
// readable.
|
||||
select {
|
||||
case it := <-w.enqueuec:
|
||||
todo = append(todo, it)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// doWork reads directories as instructed (via workc) and runs the
|
||||
// user's callback function.
|
||||
func (w *walker) doWork(wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-w.donec:
|
||||
return
|
||||
case it := <-w.workc:
|
||||
select {
|
||||
case <-w.donec:
|
||||
return
|
||||
case w.resc <- w.walk(it.dir, it.info, !it.callbackDone):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type walker struct {
|
||||
fn fs.WalkDirFunc
|
||||
|
||||
donec chan struct{} // closed on fastWalk's return
|
||||
workc chan walkItem // to workers
|
||||
enqueuec chan walkItem // from workers
|
||||
resc chan error // from workers
|
||||
|
||||
ignoredDirs []os.FileInfo
|
||||
follow bool
|
||||
}
|
||||
|
||||
type walkItem struct {
|
||||
dir string
|
||||
info fs.DirEntry
|
||||
callbackDone bool // callback already called; don't do it again
|
||||
}
|
||||
|
||||
func (w *walker) enqueue(it walkItem) {
|
||||
select {
|
||||
case w.enqueuec <- it:
|
||||
case <-w.donec:
|
||||
}
|
||||
}
|
||||
|
||||
func (w *walker) shouldSkipDir(fi os.FileInfo) bool {
|
||||
for _, ignored := range w.ignoredDirs {
|
||||
if os.SameFile(ignored, fi) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *walker) shouldTraverse(path string, de fs.DirEntry) bool {
|
||||
// TODO: do we need to use filepath.EvalSymlinks() here?
|
||||
ts, err := StatDirEntry(path, de)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if !ts.IsDir() {
|
||||
return false
|
||||
}
|
||||
if w.shouldSkipDir(ts) {
|
||||
return false
|
||||
}
|
||||
for {
|
||||
parent := filepath.Dir(path)
|
||||
if parent == path {
|
||||
return true
|
||||
}
|
||||
parentInfo, err := os.Stat(parent)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if os.SameFile(ts, parentInfo) {
|
||||
return false
|
||||
}
|
||||
path = parent
|
||||
}
|
||||
}
|
||||
|
||||
func joinPaths(dir, base string) string {
|
||||
// Handle the case where the root path argument to Walk is "/"
|
||||
// without this the returned path is prefixed with "//".
|
||||
if os.PathSeparator == '/' && dir == "/" {
|
||||
return dir + base
|
||||
}
|
||||
return dir + string(os.PathSeparator) + base
|
||||
}
|
||||
|
||||
func (w *walker) onDirEnt(dirName, baseName string, de fs.DirEntry) error {
|
||||
joined := joinPaths(dirName, baseName)
|
||||
typ := de.Type()
|
||||
if typ == os.ModeDir {
|
||||
w.enqueue(walkItem{dir: joined, info: de})
|
||||
return nil
|
||||
}
|
||||
|
||||
err := w.fn(joined, de, nil)
|
||||
if typ == os.ModeSymlink {
|
||||
if err == ErrTraverseLink {
|
||||
if !w.follow {
|
||||
// Set callbackDone so we don't call it twice for both the
|
||||
// symlink-as-symlink and the symlink-as-directory later:
|
||||
w.enqueue(walkItem{dir: joined, info: de, callbackDone: true})
|
||||
return nil
|
||||
}
|
||||
err = nil // Ignore ErrTraverseLink when Follow is true.
|
||||
}
|
||||
if err == filepath.SkipDir {
|
||||
// Permit SkipDir on symlinks too.
|
||||
return nil
|
||||
}
|
||||
if err == nil && w.follow && w.shouldTraverse(joined, de) {
|
||||
// Traverse symlink
|
||||
w.enqueue(walkItem{dir: joined, info: de, callbackDone: true})
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *walker) walk(root string, info fs.DirEntry, runUserCallback bool) error {
|
||||
if runUserCallback {
|
||||
err := w.fn(root, info, nil)
|
||||
if err == filepath.SkipDir {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err := readDir(root, w.onDirEnt)
|
||||
if err != nil {
|
||||
// Second call, to report ReadDir error.
|
||||
return w.fn(root, info, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanRootPath(root string) string {
|
||||
for i := len(root) - 1; i >= 0; i-- {
|
||||
if !os.IsPathSeparator(root[i]) {
|
||||
return root[:i+1]
|
||||
}
|
||||
}
|
||||
if root != "" {
|
||||
return root[0:1] // root is all path separators ("//")
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
//go:build darwin && go1.13 && !appengine
|
||||
// +build darwin,go1.13,!appengine
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"sync"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
//sys closedir(dir uintptr) (err error)
|
||||
//sys readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno)
|
||||
|
||||
func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error {
|
||||
if useGetdirentries {
|
||||
return readDir_Getdirentries(dirName, fn)
|
||||
} else {
|
||||
return readDir_Readir(dirName, fn)
|
||||
}
|
||||
}
|
||||
|
||||
func readDir_Readir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error {
|
||||
fd, err := opendir(dirName)
|
||||
if err != nil {
|
||||
return &os.PathError{Op: "opendir", Path: dirName, Err: err}
|
||||
}
|
||||
defer closedir(fd) //nolint:errcheck
|
||||
|
||||
skipFiles := false
|
||||
var dirent syscall.Dirent
|
||||
var entptr *syscall.Dirent
|
||||
for {
|
||||
if errno := readdir_r(fd, &dirent, &entptr); errno != 0 {
|
||||
if errno == syscall.EINTR {
|
||||
continue
|
||||
}
|
||||
return &os.PathError{Op: "readdir", Path: dirName, Err: errno}
|
||||
}
|
||||
if entptr == nil { // EOF
|
||||
break
|
||||
}
|
||||
if dirent.Ino == 0 {
|
||||
continue
|
||||
}
|
||||
typ := dtToType(dirent.Type)
|
||||
if skipFiles && typ.IsRegular() {
|
||||
continue
|
||||
}
|
||||
name := (*[len(syscall.Dirent{}.Name)]byte)(unsafe.Pointer(&dirent.Name))[:]
|
||||
for i, c := range name {
|
||||
if c == 0 {
|
||||
name = name[:i]
|
||||
break
|
||||
}
|
||||
}
|
||||
// Check for useless names before allocating a string.
|
||||
if string(name) == "." || string(name) == ".." {
|
||||
continue
|
||||
}
|
||||
nm := string(name)
|
||||
if err := fn(dirName, nm, newUnixDirent(dirName, nm, typ)); err != nil {
|
||||
if err != ErrSkipFiles {
|
||||
return err
|
||||
}
|
||||
skipFiles = true
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const direntBufSize = 32 * 1024
|
||||
|
||||
var direntBufPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
b := make([]byte, direntBufSize)
|
||||
return &b
|
||||
},
|
||||
}
|
||||
|
||||
func readDir_Getdirentries(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error {
|
||||
// This will cause the rest of the function to be omitted.
|
||||
if !useGetdirentries {
|
||||
panic("NOT IMPLEMENTED")
|
||||
}
|
||||
|
||||
fd, err := syscall.Open(dirName, syscall.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return &os.PathError{Op: "open", Path: dirName, Err: err}
|
||||
}
|
||||
defer syscall.Close(fd)
|
||||
|
||||
p := direntBufPool.Get().(*[]byte)
|
||||
defer direntBufPool.Put(p)
|
||||
dbuf := *p
|
||||
|
||||
var skipFiles bool
|
||||
var basep uintptr
|
||||
for {
|
||||
length, err := getdirentries(fd, dbuf, &basep)
|
||||
if err != nil {
|
||||
return &os.PathError{Op: "getdirentries64", Path: dirName, Err: err}
|
||||
}
|
||||
if length == 0 {
|
||||
break
|
||||
}
|
||||
buf := (*[1 << 30]byte)(unsafe.Pointer(&dbuf[0]))[:length:length]
|
||||
for len(buf) > 0 {
|
||||
dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0]))
|
||||
buf = buf[dirent.Reclen:]
|
||||
if dirent.Ino == 0 {
|
||||
continue
|
||||
}
|
||||
typ := dtToType(dirent.Type)
|
||||
if skipFiles && typ.IsRegular() {
|
||||
continue
|
||||
}
|
||||
name := (*[len(syscall.Dirent{}.Name)]byte)(unsafe.Pointer(&dirent.Name))[:]
|
||||
for i, c := range name {
|
||||
if c == 0 {
|
||||
name = name[:i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if string(name) == "." || string(name) == ".." {
|
||||
continue
|
||||
}
|
||||
nm := string(name)
|
||||
if err := fn(dirName, nm, newUnixDirent(dirName, nm, typ)); err != nil {
|
||||
if err != ErrSkipFiles {
|
||||
return err
|
||||
}
|
||||
skipFiles = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dtToType(typ uint8) os.FileMode {
|
||||
switch typ {
|
||||
case syscall.DT_BLK:
|
||||
return os.ModeDevice
|
||||
case syscall.DT_CHR:
|
||||
return os.ModeDevice | os.ModeCharDevice
|
||||
case syscall.DT_DIR:
|
||||
return os.ModeDir
|
||||
case syscall.DT_FIFO:
|
||||
return os.ModeNamedPipe
|
||||
case syscall.DT_LNK:
|
||||
return os.ModeSymlink
|
||||
case syscall.DT_REG:
|
||||
return 0
|
||||
case syscall.DT_SOCK:
|
||||
return os.ModeSocket
|
||||
}
|
||||
return ^os.FileMode(0)
|
||||
}
|
||||
|
||||
// Copied from syscall/syscall_unix.go
|
||||
|
||||
// Do the interface allocations only once for common
|
||||
// Errno values.
|
||||
var (
|
||||
errEAGAIN error = syscall.EAGAIN
|
||||
errEINVAL error = syscall.EINVAL
|
||||
errENOENT error = syscall.ENOENT
|
||||
)
|
||||
|
||||
// errnoErr returns common boxed Errno values, to prevent
|
||||
// allocations at runtime.
|
||||
func errnoErr(e syscall.Errno) error {
|
||||
switch e {
|
||||
case 0:
|
||||
return nil
|
||||
case syscall.EAGAIN:
|
||||
return errEAGAIN
|
||||
case syscall.EINVAL:
|
||||
return errEINVAL
|
||||
case syscall.ENOENT:
|
||||
return errENOENT
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd)
|
||||
// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
)
|
||||
|
||||
// readDir calls fn for each directory entry in dirName.
|
||||
// It does not descend into directories or follow symlinks.
|
||||
// If fn returns a non-nil error, readDir returns with that error
|
||||
// immediately.
|
||||
func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error {
|
||||
f, err := os.Open(dirName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
des, readErr := f.ReadDir(-1)
|
||||
f.Close()
|
||||
if readErr != nil && len(des) == 0 {
|
||||
return readErr
|
||||
}
|
||||
|
||||
var skipFiles bool
|
||||
for _, d := range des {
|
||||
if skipFiles && d.Type().IsRegular() {
|
||||
continue
|
||||
}
|
||||
// Need to use FileMode.Type().Type() for fs.DirEntry
|
||||
e := newDirEntry(dirName, d)
|
||||
if err := fn(dirName, d.Name(), e); err != nil {
|
||||
if err != ErrSkipFiles {
|
||||
return err
|
||||
}
|
||||
skipFiles = true
|
||||
}
|
||||
}
|
||||
|
||||
return readErr
|
||||
}
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build (linux || freebsd || openbsd || netbsd) && !appengine
|
||||
// +build linux freebsd openbsd netbsd
|
||||
// +build !appengine
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const blockSize = 8 << 10
|
||||
|
||||
// unknownFileMode is a sentinel (and bogus) os.FileMode
|
||||
// value used to represent a syscall.DT_UNKNOWN Dirent.Type.
|
||||
const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice
|
||||
|
||||
func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error {
|
||||
fd, err := open(dirName, 0, 0)
|
||||
if err != nil {
|
||||
return &os.PathError{Op: "open", Path: dirName, Err: err}
|
||||
}
|
||||
defer syscall.Close(fd)
|
||||
|
||||
// The buffer must be at least a block long.
|
||||
buf := make([]byte, blockSize) // stack-allocated; doesn't escape
|
||||
bufp := 0 // starting read position in buf
|
||||
nbuf := 0 // end valid data in buf
|
||||
skipFiles := false
|
||||
for {
|
||||
if bufp >= nbuf {
|
||||
bufp = 0
|
||||
nbuf, err = readDirent(fd, buf)
|
||||
if err != nil {
|
||||
return os.NewSyscallError("readdirent", err)
|
||||
}
|
||||
if nbuf <= 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
consumed, name, typ := parseDirEnt(buf[bufp:nbuf])
|
||||
bufp += consumed
|
||||
if name == "" || name == "." || name == ".." {
|
||||
continue
|
||||
}
|
||||
// Fallback for filesystems (like old XFS) that don't
|
||||
// support Dirent.Type and have DT_UNKNOWN (0) there
|
||||
// instead.
|
||||
if typ == unknownFileMode {
|
||||
fi, err := os.Lstat(dirName + "/" + name)
|
||||
if err != nil {
|
||||
// It got deleted in the meantime.
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
typ = fi.Mode() & os.ModeType
|
||||
}
|
||||
if skipFiles && typ.IsRegular() {
|
||||
continue
|
||||
}
|
||||
de := newUnixDirent(dirName, name, typ)
|
||||
if err := fn(dirName, name, de); err != nil {
|
||||
if err == ErrSkipFiles {
|
||||
skipFiles = true
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode) {
|
||||
// golang.org/issue/37269
|
||||
dirent := &syscall.Dirent{}
|
||||
copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(dirent))[:], buf)
|
||||
if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v {
|
||||
panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v))
|
||||
}
|
||||
if len(buf) < int(dirent.Reclen) {
|
||||
panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen))
|
||||
}
|
||||
consumed = int(dirent.Reclen)
|
||||
if direntInode(dirent) == 0 { // File absent in directory.
|
||||
return
|
||||
}
|
||||
switch dirent.Type {
|
||||
case syscall.DT_REG:
|
||||
typ = 0
|
||||
case syscall.DT_DIR:
|
||||
typ = os.ModeDir
|
||||
case syscall.DT_LNK:
|
||||
typ = os.ModeSymlink
|
||||
case syscall.DT_BLK:
|
||||
typ = os.ModeDevice
|
||||
case syscall.DT_FIFO:
|
||||
typ = os.ModeNamedPipe
|
||||
case syscall.DT_SOCK:
|
||||
typ = os.ModeSocket
|
||||
case syscall.DT_UNKNOWN:
|
||||
typ = unknownFileMode
|
||||
default:
|
||||
// Skip weird things.
|
||||
// It's probably a DT_WHT (http://lwn.net/Articles/325369/)
|
||||
// or something. Revisit if/when this package is moved outside
|
||||
// of goimports. goimports only cares about regular files,
|
||||
// symlinks, and directories.
|
||||
return
|
||||
}
|
||||
|
||||
nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
|
||||
nameLen := direntNamlen(dirent)
|
||||
|
||||
// Special cases for common things:
|
||||
if nameLen == 1 && nameBuf[0] == '.' {
|
||||
name = "."
|
||||
} else if nameLen == 2 && nameBuf[0] == '.' && nameBuf[1] == '.' {
|
||||
name = ".."
|
||||
} else {
|
||||
name = string(nameBuf[:nameLen])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// According to https://golang.org/doc/go1.14#runtime
|
||||
// A consequence of the implementation of preemption is that on Unix systems, including Linux and macOS
|
||||
// systems, programs built with Go 1.14 will receive more signals than programs built with earlier releases.
|
||||
//
|
||||
// This causes syscall.Open and syscall.ReadDirent sometimes fail with EINTR errors.
|
||||
// We need to retry in this case.
|
||||
func open(path string, mode int, perm uint32) (fd int, err error) {
|
||||
for {
|
||||
fd, err := syscall.Open(path, mode, perm)
|
||||
if err != syscall.EINTR {
|
||||
return fd, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readDirent(fd int, buf []byte) (n int, err error) {
|
||||
for {
|
||||
nbuf, err := syscall.ReadDirent(fd, buf)
|
||||
if err != syscall.EINTR {
|
||||
return nbuf, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build freebsd || openbsd || netbsd
|
||||
// +build freebsd openbsd netbsd
|
||||
|
||||
package fastwalk
|
||||
|
||||
import "syscall"
|
||||
|
||||
func direntNamlen(dirent *syscall.Dirent) uint64 {
|
||||
return uint64(dirent.Namlen)
|
||||
}
|
||||
|
||||
func direntInode(dirent *syscall.Dirent) uint64 {
|
||||
return uint64(dirent.Fileno)
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build linux && !appengine
|
||||
// +build linux,!appengine
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func direntNamlen(dirent *syscall.Dirent) uint64 {
|
||||
const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name))
|
||||
nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
|
||||
const nameBufLen = uint16(len(nameBuf))
|
||||
limit := dirent.Reclen - fixedHdr
|
||||
if limit > nameBufLen {
|
||||
limit = nameBufLen
|
||||
}
|
||||
for i := uint64(0); i < uint64(limit); i++ {
|
||||
if nameBuf[i] == 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
panic("failed to find terminating 0 byte in dirent")
|
||||
}
|
||||
|
||||
func direntInode(dirent *syscall.Dirent) uint64 {
|
||||
return uint64(dirent.Ino)
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
//go:build darwin && go1.12
|
||||
// +build darwin,go1.12
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Implemented in the runtime package (runtime/sys_darwin.go)
|
||||
func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno)
|
||||
func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno)
|
||||
|
||||
//go:linkname syscall_syscall syscall.syscall
|
||||
//go:linkname syscall_syscallPtr syscall.syscallPtr
|
||||
|
||||
func closedir(dir uintptr) (err error) {
|
||||
_, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, uintptr(dir), 0, 0)
|
||||
if e1 != 0 {
|
||||
err = e1
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var libc_closedir_trampoline_addr uintptr
|
||||
|
||||
//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib"
|
||||
|
||||
func readdir_r(dir uintptr, entry *syscall.Dirent, result **syscall.Dirent) syscall.Errno {
|
||||
res, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, uintptr(dir), uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result)))
|
||||
return syscall.Errno(res)
|
||||
}
|
||||
|
||||
var libc_readdir_r_trampoline_addr uintptr
|
||||
|
||||
//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib"
|
||||
|
||||
func opendir(path string) (dir uintptr, err error) {
|
||||
// We implent opendir so that we don't have to open a file, duplicate
|
||||
// it's FD, then call fdopendir with it.
|
||||
p, err := syscall.BytePtrFromString(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
r0, _, e1 := syscall_syscallPtr(libc_opendir_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0)
|
||||
dir = uintptr(r0)
|
||||
if e1 != 0 {
|
||||
err = e1
|
||||
}
|
||||
return dir, err
|
||||
}
|
||||
|
||||
var libc_opendir_trampoline_addr uintptr
|
||||
|
||||
//go:cgo_import_dynamic libc_opendir opendir "/usr/lib/libSystem.B.dylib"
|
||||
28
vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s
generated
vendored
Normal file
28
vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s
generated
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
//go:build go1.13
|
||||
// +build go1.13
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc_closedir(SB)
|
||||
|
||||
GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB)
|
||||
|
||||
TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc_readdir_r(SB)
|
||||
|
||||
GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB)
|
||||
|
||||
TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc_opendir(SB)
|
||||
|
||||
GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB)
|
||||
|
||||
TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc___getdirentries64(SB)
|
||||
|
||||
GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB)
|
||||
28
vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s
generated
vendored
Normal file
28
vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s
generated
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
//go:build go1.13
|
||||
// +build go1.13
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc_closedir(SB)
|
||||
|
||||
GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB)
|
||||
|
||||
TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc_readdir_r(SB)
|
||||
|
||||
GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB)
|
||||
|
||||
TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc_opendir(SB)
|
||||
|
||||
GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB)
|
||||
|
||||
TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0
|
||||
JMP libc___getdirentries64(SB)
|
||||
|
||||
GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8
|
||||
DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB)
|
||||
42
vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go
generated
vendored
Normal file
42
vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go
generated
vendored
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
//go:build !nogetdirentries && darwin && go1.12
|
||||
// +build !nogetdirentries,darwin,go1.12
|
||||
|
||||
package fastwalk
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const useGetdirentries = true
|
||||
|
||||
// Implemented in the runtime package (runtime/sys_darwin.go)
|
||||
func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno)
|
||||
|
||||
//go:linkname syscall_syscall6 syscall.syscall6
|
||||
|
||||
// Single-word zero for use when we need a valid pointer to 0 bytes.
|
||||
var _zero uintptr
|
||||
|
||||
func getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) {
|
||||
var _p0 unsafe.Pointer
|
||||
if len(buf) > 0 {
|
||||
_p0 = unsafe.Pointer(&buf[0])
|
||||
} else {
|
||||
_p0 = unsafe.Pointer(&_zero)
|
||||
}
|
||||
r0, _, e1 := syscall_syscall6(libc___getdirentries64_trampoline_addr,
|
||||
uintptr(fd), uintptr(_p0), uintptr(len(buf)), uintptr(unsafe.Pointer(basep)),
|
||||
0, 0)
|
||||
n = int(r0)
|
||||
if e1 != 0 {
|
||||
err = errnoErr(e1)
|
||||
} else if n < 0 {
|
||||
err = errnoErr(syscall.EINVAL)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var libc___getdirentries64_trampoline_addr uintptr
|
||||
|
||||
//go:cgo_import_dynamic libc___getdirentries64 __getdirentries64 "/usr/lib/libSystem.B.dylib"
|
||||
10
vendor/github.com/charlievieth/fastwalk/zsyscall_no_getdirentries_darwin.go
generated
vendored
Normal file
10
vendor/github.com/charlievieth/fastwalk/zsyscall_no_getdirentries_darwin.go
generated
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
//go:build nogetdirentries && darwin && go1.12
|
||||
// +build nogetdirentries,darwin,go1.12
|
||||
|
||||
package fastwalk
|
||||
|
||||
const useGetdirentries = false
|
||||
|
||||
func getdirentries(fd int, _ []byte, _ *uintptr) (int, error) {
|
||||
panic("NOT IMPLEMENTED")
|
||||
}
|
||||
|
|
@ -4,6 +4,9 @@ github.com/alecthomas/chroma/v2
|
|||
github.com/alecthomas/chroma/v2/formatters/html
|
||||
github.com/alecthomas/chroma/v2/lexers
|
||||
github.com/alecthomas/chroma/v2/styles
|
||||
# github.com/charlievieth/fastwalk v1.0.1
|
||||
## explicit; go 1.18
|
||||
github.com/charlievieth/fastwalk
|
||||
# github.com/dlclark/regexp2 v1.10.0
|
||||
## explicit; go 1.13
|
||||
github.com/dlclark/regexp2
|
||||
|
|
|
|||
Loading…
Reference in New Issue