Continue scanning instead of aborting on fs.ErrPermission or fs.ErrNotExist; add --debug flag to view output from those errors; initialize error logging earlier in setup

README.md

@@ -153,9 +153,10 @@ Flags:
       --case-sensitive        use case-sensitive matching for filters
       --code                  enable support for source code files
       --code-theme string     theme for source code syntax highlighting (default "solarized-dark256")
-      --compression string    compression format to use for index (flate, gzip, lz4, lzw, none, snappy, zlib, zstd) (default "zstd")
+      --compression string    compression format to use for index (flate, gzip, lz5, lzw, none, snappy, zlib, zstd) (default "zstd")
       --compression-fast      use fastest compression level (default is best)
-      --concurrency int       maximum concurrency for scan threads (default 8192)
+      --concurrency int       maximum concurrency for scan threads (default 9223372036854775807)
+  -d, --debug                 display even more verbose logs
       --disable-buttons       disable first/prev/next/last buttons
       --exit-on-error         shut down webserver on error, instead of just printing error
       --fallback              serve files as application/octet-stream if no matching format is registered

cmd/files.go

@@ -240,74 +240,74 @@ func hasSupportedFiles(path string, formats types.Types) (bool, error) {
 	}
 }
 
-func walkPath(path string, fileChannel chan<- string, stats *scanStats, limit chan struct{}, formats types.Types) error {
+func walkPath(path string, fileChannel chan<- string, wg0 *sync.WaitGroup, stats *scanStats, limit chan struct{}, formats types.Types, errorChannel chan<- error) {
+	defer func() {
+		wg0.Done()
+	}()
+
 	limit <- struct{}{}
 
 	defer func() {
 		<-limit
 	}()
 
-	errorChannel := make(chan error)
-	done := make(chan bool)
-
 	nodes, err := os.ReadDir(path)
 	if err != nil {
-		return err
+		stats.directoriesSkipped <- 1
+
+		errorChannel <- err
+
+		return
 	}
 
-	var directories, files = 0, 0
+	var files = 0
 
 	var skipDir = false
 
 	for _, node := range nodes {
-		if node.IsDir() {
+		if Ignore && !node.IsDir() && node.Name() == IgnoreFile {
-			directories++
-		} else {
-			if Ignore && node.Name() == IgnoreFile {
 			skipDir = true
 		}
 
 		files++
 	}
-	}
 
 	var skipFiles = false
 
-	if files <= MaxFileCount && files >= MinFileCount && !skipDir {
+	if files > MaxFileCount || files < MinFileCount || skipDir {
-		stats.directoriesMatched <- 1
-	} else {
 		stats.filesSkipped <- files
 		stats.directoriesSkipped <- 1
 
 		skipFiles = true
+	} else {
+		stats.directoriesMatched <- 1
 	}
 
-	var wg sync.WaitGroup
+	var wg1 sync.WaitGroup
 
-	wg.Add(1)
+	wg1.Add(1)
 	go func() {
-		defer wg.Done()
+		defer wg1.Done()
 		for _, node := range nodes {
-			wg.Add(1)
+			wg1.Add(1)
 
 			go func(node fs.DirEntry) {
-				defer wg.Done()
+				defer wg1.Done()
 
 				fullPath := filepath.Join(path, node.Name())
 
 				switch {
 				case node.IsDir() && Recursive:
-					err := walkPath(fullPath, fileChannel, stats, limit, formats)
+					wg0.Add(1)
-					if err != nil {
-						errorChannel <- err
 
-						return
+					walkPath(fullPath, fileChannel, wg0, stats, limit, formats, errorChannel)
-					}
 				case !node.IsDir() && !skipFiles:
 					path, err := normalizePath(fullPath)
 					if err != nil {
 						errorChannel <- err
 
+						stats.filesSkipped <- 1
+
 						return
 					}
 
@@ -325,35 +325,16 @@ func walkPath(path string, fileChannel chan<- string, stats *scanStats, limit ch
 		}
 	}()
 
-	go func() {
+	wg1.Wait()
-		wg.Wait()
-
-		time.Sleep(1 * time.Microsecond)
-
-		close(done)
-	}()
-
-Poll:
-	for {
-		select {
-		case err := <-errorChannel:
-			return err
-		case <-done:
-			break Poll
-		}
 }
 
-	return nil
+func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
-}
-
-func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types) ([]string, error) {
 	startTime := time.Now()
 
 	var filesMatched, filesSkipped int
 	var directoriesMatched, directoriesSkipped int
 
 	fileChannel := make(chan string)
-	errorChannel := make(chan error)
 	done := make(chan bool)
 
 	stats := &scanStats{
@@ -422,41 +403,19 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
 
 	limit := make(chan struct{}, Concurrency)
 
-	var wg sync.WaitGroup
+	var wg0 sync.WaitGroup
 
 	for i := 0; i < len(paths); i++ {
-		wg.Add(1)
+		wg0.Add(1)
 
 		go func(i int) {
-			defer func() {
+			walkPath(paths[i], fileChannel, &wg0, stats, limit, formats, errorChannel)
-				wg.Done()
-			}()
-
-			err := walkPath(paths[i], fileChannel, stats, limit, formats)
-
-			if err != nil {
-				errorChannel <- err
-
-				return
-			}
 		}(i)
 	}
 
-	go func() {
+	wg0.Wait()
-		wg.Wait()
 
 	close(done)
-	}()
-
-Poll:
-	for {
-		select {
-		case err := <-errorChannel:
-			return []string{}, err
-		case <-done:
-			break Poll
-		}
-	}
 
 	if Verbose {
 		fmt.Printf("%s | INDEX: Selected %d/%d files across %d/%d directories in %s\n",
@@ -471,45 +430,27 @@ Poll:
 
 	slices.Sort(list)
 
-	return list, nil
+	return list
 }
 
-func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types) ([]string, error) {
+func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
 	switch {
 	case Index && !index.isEmpty() && filters.isEmpty():
-		return index.List(), nil
+		return index.List()
 	case Index && !index.isEmpty() && !filters.isEmpty():
-		return filters.apply(index.List()), nil
+		return filters.apply(index.List())
 	case Index && index.isEmpty() && !filters.isEmpty():
-		list, err := scanPaths(paths, sort, index, formats)
+		index.set(scanPaths(paths, sort, index, formats, errorChannel))
-		if err != nil {
-			return []string{}, err
-		}
-		index.set(list)
 
-		return filters.apply(index.List()), nil
+		return filters.apply(index.List())
 	case Index && index.isEmpty() && filters.isEmpty():
-		list, err := scanPaths(paths, sort, index, formats)
+		index.set(scanPaths(paths, sort, index, formats, errorChannel))
-		if err != nil {
-			return []string{}, err
-		}
-		index.set(list)
 
-		return index.List(), nil
+		return index.List()
 	case !Index && !filters.isEmpty():
-		list, err := scanPaths(paths, sort, index, formats)
+		return filters.apply(scanPaths(paths, sort, index, formats, errorChannel))
-		if err != nil {
-			return []string{}, err
-		}
-
-		return filters.apply(list), nil
 	default:
-		list, err := scanPaths(paths, sort, index, formats)
+		return scanPaths(paths, sort, index, formats, errorChannel)
-		if err != nil {
-			return []string{}, err
-		}
-
-		return list, nil
 	}
 }
 

cmd/index.go

@@ -266,12 +266,7 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		index.clear()
 
-		_, err := fileList(args, &filters{}, "", index, formats)
+		fileList(args, &filters{}, "", index, formats, errorChannel)
-		if err != nil {
-			errorChannel <- err
-
-			return
-		}
 
 		w.Header().Set("Content-Type", "text/plain")
 
@@ -279,18 +274,13 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
 	}
 }
 
-func importIndex(args []string, index *fileIndex, formats types.Types) error {
+func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
 	if IndexFile != "" {
 		err := index.Import(IndexFile)
 		if err == nil {
-			return nil
+			return
 		}
 	}
 
-	_, err := fileList(args, &filters{}, "", index, formats)
+	fileList(args, &filters{}, "", index, formats, errorChannel)
-	if err != nil {
-		return err
-	}
-
-	return nil
 }

cmd/root.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "3.12.1"
+	ReleaseVersion    string = "4.0.0"
 )
 
 var (
@@ -31,6 +31,7 @@ var (
 	Compression     string
 	CompressionFast bool
 	Concurrency     int
+	Debug           bool
 	DisableButtons  bool
 	ExitOnError     bool
 	Fallback        bool
@@ -138,7 +139,8 @@ func init() {
 	rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
 	rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (flate, gzip, lz5, lzw, none, snappy, zlib, zstd)")
 	rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
-	rootCmd.Flags().IntVar(&Concurrency, "concurrency", 8192, "maximum concurrency for scan threads")
+	rootCmd.Flags().IntVar(&Concurrency, "concurrency", int(^uint(0)>>1), "maximum concurrency for scan threads")
+	rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
 	rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")
 	rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing error")
 	rootCmd.Flags().BoolVar(&Fallback, "fallback", false, "serve files as application/octet-stream if no matching format is registered")
@@ -172,6 +174,8 @@ func init() {
 
 	rootCmd.Flags().SetInterspersed(true)
 
+	rootCmd.MarkFlagsMutuallyExclusive("debug", "exit-on-error")
+
 	rootCmd.MarkFlagsOneRequired(RequiredArgs...)
 
 	rootCmd.SetHelpCommand(&cobra.Command{

cmd/web.go

@@ -206,14 +206,7 @@ func serveRoot(paths []string, regexes *regexes, index *fileIndex, formats types
 			}
 		}
 
-		list, err := fileList(paths, filters, sortOrder, index, formats)
+		list := fileList(paths, filters, sortOrder, index, formats, errorChannel)
-		if err != nil {
-			errorChannel <- err
-
-			serverError(w, r, nil)
-
-			return
-		}
 
 	loop:
 		for timeout := time.After(timeout); ; {
@@ -552,6 +545,31 @@ func ServePage(args []string) error {
 
 	errorChannel := make(chan error)
 
+	go func() {
+		for err := range errorChannel {
+			prefix := "ERROR"
+
+			switch {
+			case errors.Is(err, os.ErrNotExist) && Debug:
+				prefix = "DEBUG"
+			case errors.Is(err, os.ErrNotExist):
+				continue
+			case errors.Is(err, os.ErrPermission) && Debug:
+				prefix = "DEBUG"
+			case errors.Is(err, os.ErrPermission):
+				continue
+			}
+
+			fmt.Printf("%s | %s: %v\n", time.Now().Format(logDate), prefix, err)
+
+			if ExitOnError {
+				fmt.Printf("%s | %s: Shutting down...\n", time.Now().Format(logDate), prefix)
+
+				srv.Shutdown(context.Background())
+			}
+		}
+	}()
+
 	registerHandler(mux, Prefix, serveRoot(paths, regexes, index, formats, errorChannel))
 
 	Prefix = strings.TrimSuffix(Prefix, "/")
@@ -573,10 +591,7 @@ func ServePage(args []string) error {
 	if Index {
 		registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel))
 
-		err = importIndex(paths, index, formats)
+		importIndex(paths, index, formats, errorChannel)
-		if err != nil {
-			return err
-		}
 	}
 
 	if Info {
@@ -591,18 +606,6 @@ func ServePage(args []string) error {
 		fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n")
 	}
 
-	go func() {
-		for err := range errorChannel {
-			fmt.Printf("%s | ERROR: %v\n", time.Now().Format(logDate), err)
-
-			if ExitOnError {
-				fmt.Printf("%s | ERROR: Shutting down...\n", time.Now().Format(logDate))
-
-				srv.Shutdown(context.Background())
-			}
-		}
-	}()
-
 	if Verbose {
 		fmt.Printf("%s | SERVE: Listening on http://%s%s/\n",
 			time.Now().Format(logDate),