From 6f29f89acf5e71d001188df844f1bcc3ca24569e Mon Sep 17 00:00:00 2001 From: Seednode Date: Sat, 6 Jan 2024 09:41:30 -0600 Subject: [PATCH] Continue scanning instead of aborting on fs.ErrPermission or fs.ErrNotExist; add --debug flag to view output from those errors; initialize error logging earlier in setup --- README.md | 5 +- cmd/files.go | 147 +++++++++++++++------------------------------------ cmd/index.go | 18 ++----- cmd/root.go | 8 ++- cmd/web.go | 51 +++++++++--------- 5 files changed, 84 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index 52293fb..c6cba8f 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,10 @@ Flags: --case-sensitive use case-sensitive matching for filters --code enable support for source code files --code-theme string theme for source code syntax highlighting (default "solarized-dark256") - --compression string compression format to use for index (flate, gzip, lz4, lzw, none, snappy, zlib, zstd) (default "zstd") + --compression string compression format to use for index (flate, gzip, lz5, lzw, none, snappy, zlib, zstd) (default "zstd") --compression-fast use fastest compression level (default is best) - --concurrency int maximum concurrency for scan threads (default 8192) + --concurrency int maximum concurrency for scan threads (default 9223372036854775807) + -d, --debug display even more verbose logs --disable-buttons disable first/prev/next/last buttons --exit-on-error shut down webserver on error, instead of just printing error --fallback serve files as application/octet-stream if no matching format is registered diff --git a/cmd/files.go b/cmd/files.go index fe3dfe5..6544379 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -240,74 +240,74 @@ func hasSupportedFiles(path string, formats types.Types) (bool, error) { } } -func walkPath(path string, fileChannel chan<- string, stats *scanStats, limit chan struct{}, formats types.Types) error { +func walkPath(path string, fileChannel chan<- string, wg0 *sync.WaitGroup, stats *scanStats, limit chan struct{}, formats types.Types, errorChannel chan<- error) { + defer func() { + wg0.Done() + }() + limit <- struct{}{} defer func() { <-limit }() - errorChannel := make(chan error) - done := make(chan bool) - nodes, err := os.ReadDir(path) if err != nil { - return err + stats.directoriesSkipped <- 1 + + errorChannel <- err + + return } - var directories, files = 0, 0 + var files = 0 var skipDir = false for _, node := range nodes { - if node.IsDir() { - directories++ - } else { - if Ignore && node.Name() == IgnoreFile { - skipDir = true - } - - files++ + if Ignore && !node.IsDir() && node.Name() == IgnoreFile { + skipDir = true } + + files++ } var skipFiles = false - if files <= MaxFileCount && files >= MinFileCount && !skipDir { - stats.directoriesMatched <- 1 - } else { + if files > MaxFileCount || files < MinFileCount || skipDir { stats.filesSkipped <- files stats.directoriesSkipped <- 1 skipFiles = true + } else { + stats.directoriesMatched <- 1 } - var wg sync.WaitGroup + var wg1 sync.WaitGroup - wg.Add(1) + wg1.Add(1) go func() { - defer wg.Done() + defer wg1.Done() for _, node := range nodes { - wg.Add(1) + wg1.Add(1) go func(node fs.DirEntry) { - defer wg.Done() + defer wg1.Done() fullPath := filepath.Join(path, node.Name()) switch { case node.IsDir() && Recursive: - err := walkPath(fullPath, fileChannel, stats, limit, formats) - if err != nil { - errorChannel <- err + wg0.Add(1) - return - } + walkPath(fullPath, fileChannel, wg0, stats, limit, formats, errorChannel) case !node.IsDir() && !skipFiles: path, err := normalizePath(fullPath) if err != nil { errorChannel <- err + stats.filesSkipped <- 1 + return } @@ -325,35 +325,16 @@ func walkPath(path string, fileChannel chan<- string, stats *scanStats, limit ch } }() - go func() { - wg.Wait() - - time.Sleep(1 * time.Microsecond) - - close(done) - }() - -Poll: - for { - select { - case err := <-errorChannel: - return err - case <-done: - break Poll - } - } - - return nil + wg1.Wait() } -func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types) ([]string, error) { +func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string { startTime := time.Now() var filesMatched, filesSkipped int var directoriesMatched, directoriesSkipped int fileChannel := make(chan string) - errorChannel := make(chan error) done := make(chan bool) stats := &scanStats{ @@ -422,41 +403,19 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type limit := make(chan struct{}, Concurrency) - var wg sync.WaitGroup + var wg0 sync.WaitGroup for i := 0; i < len(paths); i++ { - wg.Add(1) + wg0.Add(1) go func(i int) { - defer func() { - wg.Done() - }() - - err := walkPath(paths[i], fileChannel, stats, limit, formats) - - if err != nil { - errorChannel <- err - - return - } + walkPath(paths[i], fileChannel, &wg0, stats, limit, formats, errorChannel) }(i) } - go func() { - wg.Wait() + wg0.Wait() - close(done) - }() - -Poll: - for { - select { - case err := <-errorChannel: - return []string{}, err - case <-done: - break Poll - } - } + close(done) if Verbose { fmt.Printf("%s | INDEX: Selected %d/%d files across %d/%d directories in %s\n", @@ -471,45 +430,27 @@ Poll: slices.Sort(list) - return list, nil + return list } -func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types) ([]string, error) { +func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string { switch { case Index && !index.isEmpty() && filters.isEmpty(): - return index.List(), nil + return index.List() case Index && !index.isEmpty() && !filters.isEmpty(): - return filters.apply(index.List()), nil + return filters.apply(index.List()) case Index && index.isEmpty() && !filters.isEmpty(): - list, err := scanPaths(paths, sort, index, formats) - if err != nil { - return []string{}, err - } - index.set(list) + index.set(scanPaths(paths, sort, index, formats, errorChannel)) - return filters.apply(index.List()), nil + return filters.apply(index.List()) case Index && index.isEmpty() && filters.isEmpty(): - list, err := scanPaths(paths, sort, index, formats) - if err != nil { - return []string{}, err - } - index.set(list) + index.set(scanPaths(paths, sort, index, formats, errorChannel)) - return index.List(), nil + return index.List() case !Index && !filters.isEmpty(): - list, err := scanPaths(paths, sort, index, formats) - if err != nil { - return []string{}, err - } - - return filters.apply(list), nil + return filters.apply(scanPaths(paths, sort, index, formats, errorChannel)) default: - list, err := scanPaths(paths, sort, index, formats) - if err != nil { - return []string{}, err - } - - return list, nil + return scanPaths(paths, sort, index, formats, errorChannel) } } diff --git a/cmd/index.go b/cmd/index.go index 4f4e006..3c5cf69 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -266,12 +266,7 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { index.clear() - _, err := fileList(args, &filters{}, "", index, formats) - if err != nil { - errorChannel <- err - - return - } + fileList(args, &filters{}, "", index, formats, errorChannel) w.Header().Set("Content-Type", "text/plain") @@ -279,18 +274,13 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err } } -func importIndex(args []string, index *fileIndex, formats types.Types) error { +func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { if IndexFile != "" { err := index.Import(IndexFile) if err == nil { - return nil + return } } - _, err := fileList(args, &filters{}, "", index, formats) - if err != nil { - return err - } - - return nil + fileList(args, &filters{}, "", index, formats, errorChannel) } diff --git a/cmd/root.go b/cmd/root.go index 350f80e..4deb86d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,7 @@ import ( const ( AllowedCharacters string = `^[A-z0-9.\-_]+$` - ReleaseVersion string = "3.12.1" + ReleaseVersion string = "4.0.0" ) var ( @@ -31,6 +31,7 @@ var ( Compression string CompressionFast bool Concurrency int + Debug bool DisableButtons bool ExitOnError bool Fallback bool @@ -138,7 +139,8 @@ func init() { rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting") rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (flate, gzip, lz5, lzw, none, snappy, zlib, zstd)") rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)") - rootCmd.Flags().IntVar(&Concurrency, "concurrency", 8192, "maximum concurrency for scan threads") + rootCmd.Flags().IntVar(&Concurrency, "concurrency", int(^uint(0)>>1), "maximum concurrency for scan threads") + rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs") rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons") rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing error") rootCmd.Flags().BoolVar(&Fallback, "fallback", false, "serve files as application/octet-stream if no matching format is registered") @@ -172,6 +174,8 @@ func init() { rootCmd.Flags().SetInterspersed(true) + rootCmd.MarkFlagsMutuallyExclusive("debug", "exit-on-error") + rootCmd.MarkFlagsOneRequired(RequiredArgs...) rootCmd.SetHelpCommand(&cobra.Command{ diff --git a/cmd/web.go b/cmd/web.go index 43edc4b..008944a 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -206,14 +206,7 @@ func serveRoot(paths []string, regexes *regexes, index *fileIndex, formats types } } - list, err := fileList(paths, filters, sortOrder, index, formats) - if err != nil { - errorChannel <- err - - serverError(w, r, nil) - - return - } + list := fileList(paths, filters, sortOrder, index, formats, errorChannel) loop: for timeout := time.After(timeout); ; { @@ -552,6 +545,31 @@ func ServePage(args []string) error { errorChannel := make(chan error) + go func() { + for err := range errorChannel { + prefix := "ERROR" + + switch { + case errors.Is(err, os.ErrNotExist) && Debug: + prefix = "DEBUG" + case errors.Is(err, os.ErrNotExist): + continue + case errors.Is(err, os.ErrPermission) && Debug: + prefix = "DEBUG" + case errors.Is(err, os.ErrPermission): + continue + } + + fmt.Printf("%s | %s: %v\n", time.Now().Format(logDate), prefix, err) + + if ExitOnError { + fmt.Printf("%s | %s: Shutting down...\n", time.Now().Format(logDate), prefix) + + srv.Shutdown(context.Background()) + } + } + }() + registerHandler(mux, Prefix, serveRoot(paths, regexes, index, formats, errorChannel)) Prefix = strings.TrimSuffix(Prefix, "/") @@ -573,10 +591,7 @@ func ServePage(args []string) error { if Index { registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel)) - err = importIndex(paths, index, formats) - if err != nil { - return err - } + importIndex(paths, index, formats, errorChannel) } if Info { @@ -591,18 +606,6 @@ func ServePage(args []string) error { fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n") } - go func() { - for err := range errorChannel { - fmt.Printf("%s | ERROR: %v\n", time.Now().Format(logDate), err) - - if ExitOnError { - fmt.Printf("%s | ERROR: Shutting down...\n", time.Now().Format(logDate)) - - srv.Shutdown(context.Background()) - } - } - }() - if Verbose { fmt.Printf("%s | SERVE: Listening on http://%s%s/\n", time.Now().Format(logDate),