Replace microsecond delay with another sync.WaitGroup
This commit is contained in:
parent
6074b7e546
commit
9b6d9464e6
97
cmd/files.go
97
cmd/files.go
|
@ -240,9 +240,9 @@ func hasSupportedFiles(path string, formats types.Types) (bool, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func walkPath(path string, fileChannel chan<- string, wg0 *sync.WaitGroup, stats *scanStats, limit chan struct{}, formats types.Types, errorChannel chan<- error) {
|
func walkPath(path string, fileChannel chan<- string, wg1 *sync.WaitGroup, stats *scanStats, limit chan struct{}, formats types.Types, errorChannel chan<- error) {
|
||||||
defer func() {
|
defer func() {
|
||||||
wg0.Done()
|
wg1.Done()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
limit <- struct{}{}
|
limit <- struct{}{}
|
||||||
|
@ -283,54 +283,45 @@ func walkPath(path string, fileChannel chan<- string, wg0 *sync.WaitGroup, stats
|
||||||
stats.directoriesMatched <- 1
|
stats.directoriesMatched <- 1
|
||||||
}
|
}
|
||||||
|
|
||||||
var wg1 sync.WaitGroup
|
var wg2 sync.WaitGroup
|
||||||
|
|
||||||
wg1.Add(1)
|
for _, node := range nodes {
|
||||||
go func() {
|
wg2.Add(1)
|
||||||
defer wg1.Done()
|
|
||||||
for _, node := range nodes {
|
|
||||||
wg1.Add(1)
|
|
||||||
|
|
||||||
go func(node fs.DirEntry) {
|
go func(node fs.DirEntry) {
|
||||||
defer wg1.Done()
|
defer wg2.Done()
|
||||||
|
|
||||||
fullPath := filepath.Join(path, node.Name())
|
fullPath := filepath.Join(path, node.Name())
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case node.IsDir() && Recursive:
|
case node.IsDir() && Recursive:
|
||||||
wg0.Add(1)
|
wg1.Add(1)
|
||||||
|
|
||||||
walkPath(fullPath, fileChannel, wg0, stats, limit, formats, errorChannel)
|
walkPath(fullPath, fileChannel, wg1, stats, limit, formats, errorChannel)
|
||||||
case !node.IsDir() && !skipFiles:
|
case !node.IsDir() && !skipFiles:
|
||||||
path, err := normalizePath(fullPath)
|
path, err := normalizePath(fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errorChannel <- err
|
errorChannel <- err
|
||||||
|
|
||||||
stats.filesSkipped <- 1
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if formats.Validate(path) || Fallback {
|
|
||||||
fileChannel <- path
|
|
||||||
|
|
||||||
stats.filesMatched <- 1
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
stats.filesSkipped <- 1
|
stats.filesSkipped <- 1
|
||||||
|
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}(node)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
wg1.Wait()
|
if formats.Validate(path) || Fallback {
|
||||||
|
fileChannel <- path
|
||||||
|
|
||||||
// Without this, file counts randomly vary.
|
stats.filesMatched <- 1
|
||||||
// Pending further debugging, likely relating
|
|
||||||
// to stats channel close timing issues.
|
return
|
||||||
time.Sleep(1 * time.Microsecond)
|
}
|
||||||
|
|
||||||
|
stats.filesSkipped <- 1
|
||||||
|
}
|
||||||
|
}(node)
|
||||||
|
}
|
||||||
|
|
||||||
|
wg2.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
|
func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
|
||||||
|
@ -351,7 +342,11 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
|
||||||
|
|
||||||
var list []string
|
var list []string
|
||||||
|
|
||||||
|
var wg0 sync.WaitGroup
|
||||||
|
|
||||||
|
wg0.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
defer wg0.Done()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case path := <-fileChannel:
|
case path := <-fileChannel:
|
||||||
|
@ -362,7 +357,10 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
wg0.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
defer wg0.Done()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case stat := <-stats.filesMatched:
|
case stat := <-stats.filesMatched:
|
||||||
|
@ -373,7 +371,10 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
wg0.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
defer wg0.Done()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case stat := <-stats.filesSkipped:
|
case stat := <-stats.filesSkipped:
|
||||||
|
@ -384,7 +385,10 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
wg0.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
defer wg0.Done()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case stat := <-stats.directoriesMatched:
|
case stat := <-stats.directoriesMatched:
|
||||||
|
@ -395,7 +399,10 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
wg0.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
defer wg0.Done()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case stat := <-stats.directoriesSkipped:
|
case stat := <-stats.directoriesSkipped:
|
||||||
|
@ -408,20 +415,22 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
|
||||||
|
|
||||||
limit := make(chan struct{}, Concurrency)
|
limit := make(chan struct{}, Concurrency)
|
||||||
|
|
||||||
var wg0 sync.WaitGroup
|
var wg1 sync.WaitGroup
|
||||||
|
|
||||||
for i := 0; i < len(paths); i++ {
|
for i := 0; i < len(paths); i++ {
|
||||||
wg0.Add(1)
|
wg1.Add(1)
|
||||||
|
|
||||||
go func(i int) {
|
go func(i int) {
|
||||||
walkPath(paths[i], fileChannel, &wg0, stats, limit, formats, errorChannel)
|
walkPath(paths[i], fileChannel, &wg1, stats, limit, formats, errorChannel)
|
||||||
}(i)
|
}(i)
|
||||||
}
|
}
|
||||||
|
|
||||||
wg0.Wait()
|
wg1.Wait()
|
||||||
|
|
||||||
close(done)
|
close(done)
|
||||||
|
|
||||||
|
wg0.Wait()
|
||||||
|
|
||||||
if Verbose {
|
if Verbose {
|
||||||
fmt.Printf("%s | INDEX: Selected %d/%d files across %d/%d directories in %s\n",
|
fmt.Printf("%s | INDEX: Selected %d/%d files across %d/%d directories in %s\n",
|
||||||
time.Now().Format(logDate),
|
time.Now().Format(logDate),
|
||||||
|
|
|
@ -17,7 +17,7 @@ import (
|
||||||
|
|
||||||
const (
|
const (
|
||||||
AllowedCharacters string = `^[A-z0-9.\-_]+$`
|
AllowedCharacters string = `^[A-z0-9.\-_]+$`
|
||||||
ReleaseVersion string = "5.0.1"
|
ReleaseVersion string = "5.0.2"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|
Loading…
Reference in New Issue