Remove non-zstd compression options

README.md

@@ -25,10 +25,15 @@ You can restrict access to certain functionality by prepending a secret string t
 For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`.
 
 The restricted paths are:
-- `/debug/pprof/`
+- `/debug/pprof/allocs`
+- `/debug/pprof/block`
 - `/debug/pprof/cmdline`
+- `/debug/pprof/goroutine`
+- `/debug/pprof/heap`
+- `/debug/pprof/mutex`
 - `/debug/pprof/profile`
 - `/debug/pprof/symbol`
+- `/debug/pprof/threadcreate`
 - `/debug/pprof/trace`
 - `/extensions/available`
 - `/extensions/enabled`
@@ -153,8 +158,6 @@ Flags:
       --case-sensitive        use case-sensitive matching for filters
       --code                  enable support for source code files
       --code-theme string     theme for source code syntax highlighting (default "solarized-dark256")
-      --compression string    compression format to use for index (none, zlib, zstd) (default "zstd")
-      --compression-fast      use fastest compression level (default is best)
       --concurrency int       maximum concurrency for scan threads (default 2147483647)
   -d, --debug                 display even more verbose logs
       --disable-buttons       disable first/prev/next/last buttons

cmd/errors.go

@@ -16,7 +16,6 @@ import (
 
 var (
 	ErrInvalidAdminPrefix    = errors.New("admin path must match the pattern " + AllowedCharacters)
-	ErrInvalidCompression    = errors.New("supported compression formats: none, zlib, zstd")
 	ErrInvalidConcurrency    = errors.New("concurrency limit must be a positive integer")
 	ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit")
 	ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647")

cmd/files.go

@@ -454,11 +454,11 @@ func fileList(paths []string, filters *filters, sort string, index *fileIndex, f
 	case Index && !index.isEmpty() && !filters.isEmpty():
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && !filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel))
+		index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
 
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel))
+		index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
 
 		return index.List()
 	case !Index && !filters.isEmpty():

cmd/index.go

@@ -5,7 +5,6 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 package cmd
 
 import (
-	"compress/zlib"
 	"encoding/gob"
 	"fmt"
 	"io"
@@ -63,7 +62,7 @@ func (index *fileIndex) remove(path string) {
 	index.mutex.Unlock()
 }
 
-func (index *fileIndex) set(val []string) {
+func (index *fileIndex) set(val []string, errorChannel chan<- error) {
 	length := len(val)
 
 	if length < 1 {
@@ -76,7 +75,7 @@ func (index *fileIndex) set(val []string) {
 	index.mutex.Unlock()
 
 	if Index && IndexFile != "" {
-		index.Export(IndexFile)
+		index.Export(IndexFile, errorChannel)
 	}
 }
 
@@ -94,53 +93,37 @@ func (index *fileIndex) isEmpty() bool {
 	return length == 0
 }
 
-func getReader(format string, file io.Reader) (io.ReadCloser, error) {
+func getReader(file io.Reader) (*zstd.Decoder, error) {
-	switch format {
+	reader, err := zstd.NewReader(file)
-	case "none":
+	if err != nil {
-		return io.NopCloser(file), nil
+		return nil, err
-	case "zlib":
-		return zlib.NewReader(file)
-	case "zstd":
-		reader, err := zstd.NewReader(file)
-		if err != nil {
-			return io.NopCloser(file), err
-		}
-
-		return reader.IOReadCloser(), nil
 	}
 
-	return io.NopCloser(file), ErrInvalidCompression
+	return reader, nil
 }
 
-func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
+func getWriter(file io.WriteCloser) (*zstd.Encoder, error) {
-	switch {
+	return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
-	case format == "none":
-		return file, nil
-	case format == "zlib" && CompressionFast:
-		return zlib.NewWriterLevel(file, zlib.BestSpeed)
-	case format == "zlib":
-		return zlib.NewWriterLevel(file, zlib.BestCompression)
-	case format == "zstd" && CompressionFast:
-		return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest))
-	case format == "zstd":
-		return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
-	}
-
-	return file, ErrInvalidCompression
 }
 
-func (index *fileIndex) Export(path string) error {
+func (index *fileIndex) Export(path string, errorChannel chan<- error) {
 	startTime := time.Now()
 
 	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 	defer file.Close()
 
-	encoder, err := getWriter(Compression, file)
+	encoder, err := getWriter(file)
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		encoder.Close()
+
+		return
 	}
 	defer encoder.Close()
 
@@ -151,22 +134,22 @@ func (index *fileIndex) Export(path string) error {
 	if err != nil {
 		index.mutex.RUnlock()
 
-		return err
+		errorChannel <- err
+
+		return
 	}
 	length := len(index.list)
 	index.mutex.RUnlock()
 
 	// Close encoder prior to checking file size,
 	// to ensure the correct value is returned.
-	// If no compression is used, skip this step,
+	encoder.Close()
-	// as the encoder is just the file itself.
-	if Compression != "none" {
-		encoder.Close()
-	}
 
 	stats, err := file.Stat()
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 
 	if Verbose {
@@ -178,27 +161,31 @@ func (index *fileIndex) Export(path string) error {
 			time.Since(startTime).Round(time.Microsecond),
 		)
 	}
-
-	return nil
 }
 
-func (index *fileIndex) Import(path string) error {
+func (index *fileIndex) Import(path string, errorChannel chan<- error) {
 	startTime := time.Now()
 
 	file, err := os.OpenFile(path, os.O_RDONLY, 0600)
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 	defer file.Close()
 
 	stats, err := file.Stat()
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 
-	reader, err := getReader(Compression, file)
+	reader, err := getReader(file)
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 	defer reader.Close()
 
@@ -209,7 +196,9 @@ func (index *fileIndex) Import(path string) error {
 	if err != nil {
 		index.mutex.Unlock()
 
-		return err
+		errorChannel <- err
+
+		return
 	}
 	length := len(index.list)
 	index.mutex.Unlock()
@@ -223,8 +212,6 @@ func (index *fileIndex) Import(path string) error {
 			time.Since(startTime).Round(time.Microsecond),
 		)
 	}
-
-	return nil
 }
 
 func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
@@ -256,10 +243,7 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
 
 func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
 	if IndexFile != "" {
-		err := index.Import(IndexFile)
+		index.Import(IndexFile, errorChannel)
-		if err == nil {
-			return
-		}
 	}
 
 	fileList(args, &filters{}, "", index, formats, errorChannel)

cmd/root.go

@@ -10,7 +10,6 @@ import (
 	"math"
 	"os"
 	"regexp"
-	"slices"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -18,51 +17,49 @@ import (
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "5.3.0"
+	ReleaseVersion    string = "5.4.0"
 )
 
 var (
-	AdminPrefix     string
+	AdminPrefix    string
-	All             bool
+	All            bool
-	AllowEmpty      bool
+	AllowEmpty     bool
-	Audio           bool
+	Audio          bool
-	BinaryPrefix    bool
+	BinaryPrefix   bool
-	Bind            string
+	Bind           string
-	CaseSensitive   bool
+	CaseSensitive  bool
-	Code            bool
+	Code           bool
-	CodeTheme       string
+	CodeTheme      string
-	Compression     string
+	Concurrency    int
-	CompressionFast bool
+	Debug          bool
-	Concurrency     int
+	DisableButtons bool
-	Debug           bool
+	ExitOnError    bool
-	DisableButtons  bool
+	Fallback       bool
-	ExitOnError     bool
+	Filtering      bool
-	Fallback        bool
+	Flash          bool
-	Filtering       bool
+	Fun            bool
-	Flash           bool
+	Handlers       bool
-	Fun             bool
+	Ignore         bool
-	Handlers        bool
+	IgnoreFile     string
-	Ignore          bool
+	Images         bool
-	IgnoreFile      string
+	Index          bool
-	Images          bool
+	IndexFile      string
-	Index           bool
+	Info           bool
-	IndexFile       string
+	MaxFileCount   int
-	Info            bool
+	MinFileCount   int
-	MaxFileCount    int
+	PageLength     int
-	MinFileCount    int
+	Port           int
-	PageLength      int
+	Prefix         string
-	Port            int
+	Profile        bool
-	Prefix          string
+	Recursive      bool
-	Profile         bool
+	Redact         bool
-	Recursive       bool
+	Refresh        bool
-	Redact          bool
+	Russian        bool
-	Refresh         bool
+	Sorting        bool
-	Russian         bool
+	Text           bool
-	Sorting         bool
+	Verbose        bool
-	Text            bool
+	Version        bool
-	Verbose         bool
+	Videos         bool
-	Version         bool
-	Videos          bool
 
 	RequiredArgs = []string{
 		"all",
@@ -91,8 +88,6 @@ var (
 				return ErrInvalidConcurrency
 			case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
 				return ErrInvalidIgnoreFile
-			case !slices.Contains(CompressionFormats, Compression):
-				return ErrInvalidCompression
 			case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
 				return ErrInvalidAdminPrefix
 			case AdminPrefix != "":
@@ -131,8 +126,6 @@ func init() {
 	rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
 	rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
 	rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
-	rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (none, zlib, zstd)")
-	rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
 	rootCmd.Flags().IntVar(&Concurrency, "concurrency", 10240, "maximum concurrency for scan threads")
 	rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
 	rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")