Rewrite filewalk functions, add (incomplete) to logging when -v|--verbose is passed and a broken pipe error is received

cmd/files.go

@@ -13,7 +13,6 @@ import (
 	"crypto/rand"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,11 +34,6 @@ type regexes struct {
 	filename     *regexp.Regexp
 }
 
-type concurrency struct {
-	directoryScans chan int
-	fileScans      chan int
-}
-
 type scanStats struct {
 	filesMatched       int
 	filesSkipped       int
@@ -47,6 +41,13 @@ type scanStats struct {
 	directoriesSkipped int
 }
 
+type scanStatsChannels struct {
+	filesMatched       chan int
+	filesSkipped       chan int
+	directoriesMatched chan int
+	directoriesSkipped chan int
+}
+
 type splitPath struct {
 	base      string
 	number    int
@@ -92,16 +93,30 @@ func kill(path string, cache *fileCache) error {
 	return nil
 }
 
-func preparePath(path string) string {
+func split(path string, regexes *regexes) (*splitPath, error) {
-	if runtime.GOOS == "windows" {
+	p := splitPath{}
-		return fmt.Sprintf("%s/%s", mediaPrefix, filepath.ToSlash(path))
+	var err error
+
+	split := regexes.filename.FindAllStringSubmatch(path, -1)
+
+	if len(split) < 1 || len(split[0]) < 3 {
+		return &splitPath{}, nil
 	}
 
-	return mediaPrefix + path
+	p.base = split[0][1]
+
+	p.number, err = strconv.Atoi(split[0][2])
+	if err != nil {
+		return &splitPath{}, err
+	}
+
+	p.extension = split[0][3]
+
+	return &p, nil
 }
 
-func newFile(paths []string, filters *filters, sortOrder string, regexes *regexes, cache *fileCache, formats *types.Types) (string, error) {
+func newFile(list []string, sortOrder string, regexes *regexes, formats *types.Types) (string, error) {
-	path, err := pickFile(paths, filters, sortOrder, cache, formats)
+	path, err := pickFile(list)
 	if err != nil {
 		return "", err
 	}
@@ -167,28 +182,6 @@ func nextFile(path, sortOrder string, regexes *regexes, formats *types.Types) (s
 	return fileName, err
 }
 
-func split(path string, regexes *regexes) (*splitPath, error) {
-	p := splitPath{}
-	var err error
-
-	split := regexes.filename.FindAllStringSubmatch(path, -1)
-
-	if len(split) < 1 || len(split[0]) < 3 {
-		return &splitPath{}, nil
-	}
-
-	p.base = split[0][1]
-
-	p.number, err = strconv.Atoi(split[0][2])
-	if err != nil {
-		return &splitPath{}, err
-	}
-
-	p.extension = split[0][3]
-
-	return &p, nil
-}
-
 func tryExtensions(splitPath *splitPath, formats *types.Types) (string, error) {
 	var fileName string
 
@@ -295,49 +288,42 @@ func pathCount(path string) (int, int, error) {
 	return files, directories, nil
 }
 
-func scanPath(path string, fileChannel chan<- string, statChannel chan<- *scanStats, errorChannel chan<- error, concurrency *concurrency, formats *types.Types) {
+func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats *scanStatsChannels, formats *types.Types) error {
 	var wg sync.WaitGroup
 
-	stats := &scanStats{
+	errorChannel := make(chan error)
-		filesMatched:       0,
+	done := make(chan bool, 1)
-		filesSkipped:       0,
-		directoriesMatched: 0,
-		directoriesSkipped: 0,
-	}
-
-	err := filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
 
+	filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error {
 		switch {
 		case !Recursive && info.IsDir() && p != path:
 			return filepath.SkipDir
 		case !info.IsDir():
 			wg.Add(1)
-			concurrency.fileScans <- 1
+			fileScans <- 1
 
 			go func() {
 				defer func() {
-					<-concurrency.fileScans
-
 					wg.Done()
+					<-fileScans
 				}()
 
 				path, err := normalizePath(p)
 				if err != nil {
 					errorChannel <- err
+
+					return
 				}
 
 				if !formats.Validate(path) {
-					stats.filesSkipped = stats.filesSkipped + 1
+					stats.filesSkipped <- 1
 
 					return
 				}
 
 				fileChannel <- path
 
-				stats.filesMatched = stats.filesMatched + 1
+				stats.filesMatched <- 1
 			}()
 		case info.IsDir():
 			files, directories, err := pathCount(p)
@@ -347,33 +333,43 @@ func scanPath(path string, fileChannel chan<- string, statChannel chan<- *scanSt
 
 			if files > 0 && (files < int(MinimumFileCount)) || (files > int(MaximumFileCount)) {
 				// This count will not otherwise include the parent directory itself, so increment by one
-				stats.directoriesSkipped = stats.directoriesSkipped + directories + 1
+				stats.directoriesSkipped <- directories + 1
-				stats.filesSkipped = stats.filesSkipped + files
+				stats.filesSkipped <- files
 
 				return filepath.SkipDir
 			}
 
-			stats.directoriesMatched = stats.directoriesMatched + 1
+			stats.directoriesMatched <- 1
 		}
 
-		return err
+		return nil
 	})
 
+	go func() {
 		wg.Wait()
+		done <- true
+	}()
 
-	statChannel <- stats
+Poll:
-
+	for {
-	if err != nil {
+		select {
-		errorChannel <- err
+		case e := <-errorChannel:
+			return e
+		case <-done:
+			break Poll
 		}
+	}
+
+	return nil
 }
 
 func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Types) ([]string, error) {
 	var list []string
 
 	fileChannel := make(chan string)
-	statChannel := make(chan *scanStats)
 	errorChannel := make(chan error)
+	directoryScans := make(chan int, maxDirectoryScans)
+	fileScans := make(chan int, maxFileScans)
 	done := make(chan bool, 1)
 
 	stats := &scanStats{
@@ -383,9 +379,11 @@ func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Typ
 		directoriesSkipped: 0,
 	}
 
-	concurrency := &concurrency{
+	statsChannels := &scanStatsChannels{
-		directoryScans: make(chan int, maxDirectoryScans),
+		filesMatched:       make(chan int),
-		fileScans:      make(chan int, maxFileScans),
+		filesSkipped:       make(chan int),
+		directoriesMatched: make(chan int),
+		directoriesSkipped: make(chan int),
 	}
 
 	var wg sync.WaitGroup
@@ -394,16 +392,20 @@ func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Typ
 
 	for i := 0; i < len(paths); i++ {
 		wg.Add(1)
-		concurrency.directoryScans <- 1
+		directoryScans <- 1
 
 		go func(i int) {
 			defer func() {
-				<-concurrency.directoryScans
-
 				wg.Done()
+				<-directoryScans
 			}()
 
-			scanPath(paths[i], fileChannel, statChannel, errorChannel, concurrency, formats)
+			err := walkPath(paths[i], fileChannel, fileScans, statsChannels, formats)
+			if err != nil {
+				errorChannel <- err
+
+				return
+			}
 		}(i)
 	}
 
@@ -417,11 +419,14 @@ Poll:
 		select {
 		case p := <-fileChannel:
 			list = append(list, p)
-		case s := <-statChannel:
+		case s := <-statsChannels.filesMatched:
-			stats.filesMatched = stats.filesMatched + s.filesMatched
+			stats.filesMatched = stats.filesMatched + s
-			stats.filesSkipped = stats.filesSkipped + s.filesSkipped
+		case s := <-statsChannels.filesSkipped:
-			stats.directoriesMatched = stats.directoriesMatched + s.directoriesMatched
+			stats.filesSkipped = stats.filesSkipped + s
-			stats.directoriesSkipped = stats.directoriesSkipped + s.directoriesSkipped
+		case s := <-statsChannels.directoriesMatched:
+			stats.directoriesMatched = stats.directoriesMatched + s
+		case s := <-statsChannels.directoriesSkipped:
+			stats.directoriesSkipped = stats.directoriesSkipped + s
 		case e := <-errorChannel:
 			return []string{}, e
 		case <-done:
@@ -487,12 +492,7 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f
 	}
 }
 
-func pickFile(args []string, filters *filters, sort string, cache *fileCache, formats *types.Types) (string, error) {
+func pickFile(list []string) (string, error) {
-	list, err := fileList(args, filters, sort, cache, formats)
-	if err != nil {
-		return "", err
-	}
-
 	fileCount := len(list)
 
 	if fileCount < 1 {

cmd/root.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	ReleaseVersion string = "0.90.3"
+	ReleaseVersion string = "0.91.0"
 )
 
 var (

cmd/web.go

@@ -19,6 +19,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/julienschmidt/httprouter"
@@ -40,6 +41,14 @@ const (
 	timeout            time.Duration = 10 * time.Second
 )
 
+func preparePath(path string) string {
+	if runtime.GOOS == "windows" {
+		return fmt.Sprintf("%s/%s", mediaPrefix, filepath.ToSlash(path))
+	}
+
+	return mediaPrefix + path
+}
+
 func serveStaticFile(paths []string, cache *fileCache, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		prefix := Prefix + sourcePrefix
@@ -96,7 +105,19 @@ func serveStaticFile(paths []string, cache *fileCache, errorChannel chan<- error
 			return
 		}
 
-		written, _ := w.Write(buf)
+		var status string
+
+		written, err := w.Write(buf)
+		switch {
+		case errors.Is(err, syscall.EPIPE):
+			status = " (incomplete)"
+		case err != nil:
+			errorChannel <- err
+
+			serverError(w, r, nil)
+
+			return
+		}
 
 		refererUri, err := stripQueryParams(refererToUri(r.Referer()))
 		if err != nil {
@@ -119,12 +140,13 @@ func serveStaticFile(paths []string, cache *fileCache, errorChannel chan<- error
 		}
 
 		if Verbose {
-			fmt.Printf("%s | Serve: %s (%s) to %s in %s\n",
+			fmt.Printf("%s | Serve: %s (%s) to %s in %s%s\n",
 				startTime.Format(logDate),
 				filePath,
 				humanReadableSize(written),
 				realIP(r),
 				time.Since(startTime).Round(time.Microsecond),
+				status,
 			)
 		}
 	}
@@ -152,10 +174,10 @@ func serveRoot(paths []string, regexes *regexes, cache *fileCache, formats *type
 
 		_, refreshInterval := refreshInterval(r)
 
-		var filePath string
+		var path string
 
 		if refererUri != "" {
-			filePath, err = nextFile(strippedRefererUri, sortOrder, regexes, formats)
+			path, err = nextFile(strippedRefererUri, sortOrder, regexes, formats)
 			if err != nil {
 				errorChannel <- err
 
@@ -165,6 +187,15 @@ func serveRoot(paths []string, regexes *regexes, cache *fileCache, formats *type
 			}
 		}
 
+		list, err := fileList(paths, filters, sortOrder, cache, formats)
+		if err != nil {
+			errorChannel <- err
+
+			serverError(w, r, nil)
+
+			return
+		}
+
 	loop:
 		for timeout := time.After(timeout); ; {
 			select {
@@ -173,14 +204,14 @@ func serveRoot(paths []string, regexes *regexes, cache *fileCache, formats *type
 			default:
 			}
 
-			if filePath != "" {
+			if path != "" {
 				break loop
 			}
 
-			filePath, err = newFile(paths, filters, sortOrder, regexes, cache, formats)
+			path, err = newFile(list, sortOrder, regexes, formats)
 			switch {
 			case err != nil && err == ErrNoMediaFound:
-				notFound(w, r, filePath)
+				notFound(w, r, path)
 
 				return
 			case err != nil:
@@ -197,7 +228,7 @@ func serveRoot(paths []string, regexes *regexes, cache *fileCache, formats *type
 		newUrl := fmt.Sprintf("http://%s%s%s%s",
 			r.Host,
 			Prefix,
-			preparePath(filePath),
+			preparePath(path),
 			queryParams,
 		)
 		http.Redirect(w, r, newUrl, RedirectStatusCode)