diff --git a/README.md b/README.md index 046e267..e15cd4f 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,15 @@ You can restrict access to certain functionality by prepending a secret string t For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`. The restricted paths are: -- `/debug/pprof/` +- `/debug/pprof/allocs` +- `/debug/pprof/block` - `/debug/pprof/cmdline` +- `/debug/pprof/goroutine` +- `/debug/pprof/heap` +- `/debug/pprof/mutex` - `/debug/pprof/profile` - `/debug/pprof/symbol` +- `/debug/pprof/threadcreate` - `/debug/pprof/trace` - `/extensions/available` - `/extensions/enabled` @@ -153,8 +158,6 @@ Flags: --case-sensitive use case-sensitive matching for filters --code enable support for source code files --code-theme string theme for source code syntax highlighting (default "solarized-dark256") - --compression string compression format to use for index (none, zlib, zstd) (default "zstd") - --compression-fast use fastest compression level (default is best) --concurrency int maximum concurrency for scan threads (default 2147483647) -d, --debug display even more verbose logs --disable-buttons disable first/prev/next/last buttons diff --git a/cmd/errors.go b/cmd/errors.go index 3eaa9b9..4bf4477 100644 --- a/cmd/errors.go +++ b/cmd/errors.go @@ -16,7 +16,6 @@ import ( var ( ErrInvalidAdminPrefix = errors.New("admin path must match the pattern " + AllowedCharacters) - ErrInvalidCompression = errors.New("supported compression formats: none, zlib, zstd") ErrInvalidConcurrency = errors.New("concurrency limit must be a positive integer") ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit") ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647") diff --git a/cmd/files.go b/cmd/files.go index cdb9a35..605cbda 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -454,11 +454,11 @@ func fileList(paths []string, filters *filters, sort string, index *fileIndex, f case Index && !index.isEmpty() && !filters.isEmpty(): return filters.apply(index.List()) case Index && index.isEmpty() && !filters.isEmpty(): - index.set(scanPaths(paths, sort, index, formats, errorChannel)) + index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel) return filters.apply(index.List()) case Index && index.isEmpty() && filters.isEmpty(): - index.set(scanPaths(paths, sort, index, formats, errorChannel)) + index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel) return index.List() case !Index && !filters.isEmpty(): diff --git a/cmd/index.go b/cmd/index.go index 7518817..6583e76 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -5,7 +5,6 @@ Copyright © 2023 Seednode package cmd import ( - "compress/zlib" "encoding/gob" "fmt" "io" @@ -63,7 +62,7 @@ func (index *fileIndex) remove(path string) { index.mutex.Unlock() } -func (index *fileIndex) set(val []string) { +func (index *fileIndex) set(val []string, errorChannel chan<- error) { length := len(val) if length < 1 { @@ -76,7 +75,7 @@ func (index *fileIndex) set(val []string) { index.mutex.Unlock() if Index && IndexFile != "" { - index.Export(IndexFile) + index.Export(IndexFile, errorChannel) } } @@ -94,53 +93,37 @@ func (index *fileIndex) isEmpty() bool { return length == 0 } -func getReader(format string, file io.Reader) (io.ReadCloser, error) { - switch format { - case "none": - return io.NopCloser(file), nil - case "zlib": - return zlib.NewReader(file) - case "zstd": - reader, err := zstd.NewReader(file) - if err != nil { - return io.NopCloser(file), err - } - - return reader.IOReadCloser(), nil +func getReader(file io.Reader) (*zstd.Decoder, error) { + reader, err := zstd.NewReader(file) + if err != nil { + return nil, err } - return io.NopCloser(file), ErrInvalidCompression + return reader, nil } -func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) { - switch { - case format == "none": - return file, nil - case format == "zlib" && CompressionFast: - return zlib.NewWriterLevel(file, zlib.BestSpeed) - case format == "zlib": - return zlib.NewWriterLevel(file, zlib.BestCompression) - case format == "zstd" && CompressionFast: - return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest)) - case format == "zstd": - return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) - } - - return file, ErrInvalidCompression +func getWriter(file io.WriteCloser) (*zstd.Encoder, error) { + return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) } -func (index *fileIndex) Export(path string) error { +func (index *fileIndex) Export(path string, errorChannel chan<- error) { startTime := time.Now() file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { - return err + errorChannel <- err + + return } defer file.Close() - encoder, err := getWriter(Compression, file) + encoder, err := getWriter(file) if err != nil { - return err + errorChannel <- err + + encoder.Close() + + return } defer encoder.Close() @@ -151,22 +134,22 @@ func (index *fileIndex) Export(path string) error { if err != nil { index.mutex.RUnlock() - return err + errorChannel <- err + + return } length := len(index.list) index.mutex.RUnlock() // Close encoder prior to checking file size, // to ensure the correct value is returned. - // If no compression is used, skip this step, - // as the encoder is just the file itself. - if Compression != "none" { - encoder.Close() - } + encoder.Close() stats, err := file.Stat() if err != nil { - return err + errorChannel <- err + + return } if Verbose { @@ -178,27 +161,31 @@ func (index *fileIndex) Export(path string) error { time.Since(startTime).Round(time.Microsecond), ) } - - return nil } -func (index *fileIndex) Import(path string) error { +func (index *fileIndex) Import(path string, errorChannel chan<- error) { startTime := time.Now() file, err := os.OpenFile(path, os.O_RDONLY, 0600) if err != nil { - return err + errorChannel <- err + + return } defer file.Close() stats, err := file.Stat() if err != nil { - return err + errorChannel <- err + + return } - reader, err := getReader(Compression, file) + reader, err := getReader(file) if err != nil { - return err + errorChannel <- err + + return } defer reader.Close() @@ -209,7 +196,9 @@ func (index *fileIndex) Import(path string) error { if err != nil { index.mutex.Unlock() - return err + errorChannel <- err + + return } length := len(index.list) index.mutex.Unlock() @@ -223,8 +212,6 @@ func (index *fileIndex) Import(path string) error { time.Since(startTime).Round(time.Microsecond), ) } - - return nil } func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle { @@ -256,10 +243,7 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { if IndexFile != "" { - err := index.Import(IndexFile) - if err == nil { - return - } + index.Import(IndexFile, errorChannel) } fileList(args, &filters{}, "", index, formats, errorChannel) diff --git a/cmd/root.go b/cmd/root.go index 3cd8a7b..e8bbe0c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,7 +10,6 @@ import ( "math" "os" "regexp" - "slices" "time" "github.com/spf13/cobra" @@ -18,51 +17,49 @@ import ( const ( AllowedCharacters string = `^[A-z0-9.\-_]+$` - ReleaseVersion string = "5.3.0" + ReleaseVersion string = "5.4.0" ) var ( - AdminPrefix string - All bool - AllowEmpty bool - Audio bool - BinaryPrefix bool - Bind string - CaseSensitive bool - Code bool - CodeTheme string - Compression string - CompressionFast bool - Concurrency int - Debug bool - DisableButtons bool - ExitOnError bool - Fallback bool - Filtering bool - Flash bool - Fun bool - Handlers bool - Ignore bool - IgnoreFile string - Images bool - Index bool - IndexFile string - Info bool - MaxFileCount int - MinFileCount int - PageLength int - Port int - Prefix string - Profile bool - Recursive bool - Redact bool - Refresh bool - Russian bool - Sorting bool - Text bool - Verbose bool - Version bool - Videos bool + AdminPrefix string + All bool + AllowEmpty bool + Audio bool + BinaryPrefix bool + Bind string + CaseSensitive bool + Code bool + CodeTheme string + Concurrency int + Debug bool + DisableButtons bool + ExitOnError bool + Fallback bool + Filtering bool + Flash bool + Fun bool + Handlers bool + Ignore bool + IgnoreFile string + Images bool + Index bool + IndexFile string + Info bool + MaxFileCount int + MinFileCount int + PageLength int + Port int + Prefix string + Profile bool + Recursive bool + Redact bool + Refresh bool + Russian bool + Sorting bool + Text bool + Verbose bool + Version bool + Videos bool RequiredArgs = []string{ "all", @@ -91,8 +88,6 @@ var ( return ErrInvalidConcurrency case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile): return ErrInvalidIgnoreFile - case !slices.Contains(CompressionFormats, Compression): - return ErrInvalidCompression case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix): return ErrInvalidAdminPrefix case AdminPrefix != "": @@ -131,8 +126,6 @@ func init() { rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters") rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files") rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting") - rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (none, zlib, zstd)") - rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)") rootCmd.Flags().IntVar(&Concurrency, "concurrency", 10240, "maximum concurrency for scan threads") rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs") rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")