Split scanPaths stat counter incrementing into separate goroutines

This commit is contained in:
Seednode 2023-10-04 21:54:06 -05:00
parent 9844f4a2e1
commit 17abb25e74
2 changed files with 54 additions and 31 deletions

View File

@ -27,12 +27,12 @@ type regexes struct {
filename *regexp.Regexp filename *regexp.Regexp
} }
type scanStats struct { // type scanStats struct {
filesMatched int // filesMatched int
filesSkipped int // filesSkipped int
directoriesMatched int // directoriesMatched int
directoriesSkipped int // directoriesSkipped int
} // }
type scanStatsChannels struct { type scanStatsChannels struct {
filesMatched chan int filesMatched chan int
@ -322,19 +322,21 @@ Poll:
} }
func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types) ([]string, error) { func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types) ([]string, error) {
startTime := time.Now()
var list []string var list []string
var filesMatched int = 0
var filesSkipped int = 0
var directoriesMatched int = 0
var directoriesSkipped int = 0
var wg sync.WaitGroup
fileChannel := make(chan string) fileChannel := make(chan string)
errorChannel := make(chan error) errorChannel := make(chan error)
done := make(chan bool, 1) done := make(chan bool, 1)
stats := &scanStats{
filesMatched: 0,
filesSkipped: 0,
directoriesMatched: 0,
directoriesSkipped: 0,
}
statsChannels := &scanStatsChannels{ statsChannels := &scanStatsChannels{
filesMatched: make(chan int), filesMatched: make(chan int),
filesSkipped: make(chan int), filesSkipped: make(chan int),
@ -342,9 +344,35 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
directoriesSkipped: make(chan int), directoriesSkipped: make(chan int),
} }
var wg sync.WaitGroup go func() {
for path := range fileChannel {
list = append(list, path)
}
}()
startTime := time.Now() go func() {
for stat := range statsChannels.filesMatched {
filesMatched += stat
}
}()
go func() {
for stat := range statsChannels.filesSkipped {
filesSkipped += stat
}
}()
go func() {
for stat := range statsChannels.directoriesMatched {
directoriesMatched += stat
}
}()
go func() {
for stat := range statsChannels.directoriesSkipped {
directoriesSkipped += stat
}
}()
for i := 0; i < len(paths); i++ { for i := 0; i < len(paths); i++ {
wg.Add(1) wg.Add(1)
@ -367,21 +395,16 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
wg.Wait() wg.Wait()
done <- true done <- true
close(statsChannels.filesMatched)
close(statsChannels.filesSkipped)
close(statsChannels.directoriesMatched)
close(statsChannels.directoriesSkipped)
}() }()
Poll: Poll:
for { for {
select { select {
case path := <-fileChannel:
list = append(list, path)
case stat := <-statsChannels.filesMatched:
stats.filesMatched += stat
case stat := <-statsChannels.filesSkipped:
stats.filesSkipped += stat
case stat := <-statsChannels.directoriesMatched:
stats.directoriesMatched += stat
case stat := <-statsChannels.directoriesSkipped:
stats.directoriesSkipped += stat
case err := <-errorChannel: case err := <-errorChannel:
return []string{}, err return []string{}, err
case <-done: case <-done:
@ -389,7 +412,7 @@ Poll:
} }
} }
if stats.filesMatched < 1 { if filesMatched < 1 {
fmt.Println("No files matched") fmt.Println("No files matched")
return []string{}, nil return []string{}, nil
@ -398,10 +421,10 @@ Poll:
if Verbose { if Verbose {
fmt.Printf("%s | INDEX: Selecting from %d/%d files across %d/%d directories in %s\n", fmt.Printf("%s | INDEX: Selecting from %d/%d files across %d/%d directories in %s\n",
time.Now().Format(logDate), time.Now().Format(logDate),
stats.filesMatched, filesMatched,
stats.filesMatched+stats.filesSkipped, filesMatched+filesSkipped,
stats.directoriesMatched, directoriesMatched,
stats.directoriesMatched+stats.directoriesSkipped, directoriesMatched+directoriesSkipped,
time.Since(startTime), time.Since(startTime),
) )
} }

View File

@ -12,7 +12,7 @@ import (
) )
const ( const (
ReleaseVersion string = "2.6.0" ReleaseVersion string = "2.7.0"
) )
var ( var (