Add --compression, to allow specifying compression format for index file

README.md

@@ -145,9 +145,10 @@ Flags:
       --case-sensitive        use case-sensitive matching for filters
       --code                  enable support for source code files
       --code-theme string     theme for source code syntax highlighting (default "solarized-dark256")
+      --compression string    compression format to use for index (flate, gzip, lzw, none, zlib, zstd) (default "zstd")
       --concurrency int       maximum concurrency for scan threads (default 8192)
       --disable-buttons       disable first/prev/next/last buttons
-      --exit-on-error         shut down webserver on error, instead of just printing the error
+      --exit-on-error         shut down webserver on error, instead of just printing error
       --fallback              serve files as application/octet-stream if no matching format is registered
   -f, --filter                enable filtering
       --flash                 enable support for shockwave flash files (via ruffle.rs)

cmd/errors.go

@@ -16,6 +16,7 @@ import (
 
 var (
 	ErrInvalidAdminPrefix    = errors.New("admin path must match the pattern " + AllowedCharacters)
+	ErrInvalidCompression    = errors.New("supported compression formats: flate, gzip, lzw, none, zlib, zstd")
 	ErrInvalidConcurrency    = errors.New("concurrency limit must be a positive integer")
 	ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit")
 	ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647")

cmd/index.go

@@ -5,8 +5,13 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 package cmd
 
 import (
+	"compress/flate"
+	"compress/gzip"
+	"compress/lzw"
+	"compress/zlib"
 	"encoding/gob"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"sync"
@@ -86,6 +91,46 @@ func (index *fileIndex) isEmpty() bool {
 	return length == 0
 }
 
+func getReader(format string, file io.Reader) (io.ReadCloser, error) {
+	switch format {
+	case "flate":
+		return flate.NewReader(file), nil
+	case "gzip":
+		return gzip.NewReader(file)
+	case "lzw":
+		return lzw.NewReader(file, lzw.LSB, 8), nil
+	case "none":
+		return io.NopCloser(file), nil
+	case "zlib":
+		return zlib.NewReader(file)
+	case "zstd":
+		decoder, err := zstd.NewReader(file)
+
+		return decoder.IOReadCloser(), err
+	}
+
+	return io.NopCloser(file), ErrInvalidCompression
+}
+
+func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
+	switch format {
+	case "flate":
+		return flate.NewWriter(file, flate.DefaultCompression)
+	case "gzip":
+		return gzip.NewWriter(file), nil
+	case "lzw":
+		return lzw.NewWriter(file, lzw.LSB, 8), nil
+	case "none":
+		return file, nil
+	case "zlib":
+		return zlib.NewWriter(file), nil
+	case "zstd":
+		return zstd.NewWriter(file)
+	}
+
+	return file, ErrInvalidCompression
+}
+
 func (index *fileIndex) Export(path string) error {
 	startTime := time.Now()
 
@@ -95,13 +140,13 @@ func (index *fileIndex) Export(path string) error {
 	}
 	defer file.Close()
 
-	z, err := zstd.NewWriter(file)
+	encoder, err := getWriter(Compression, file)
 	if err != nil {
 		return err
 	}
-	defer z.Close()
+	defer encoder.Close()
 
-	enc := gob.NewEncoder(z)
+	enc := gob.NewEncoder(encoder)
 
 	index.mutex.RLock()
 	err = enc.Encode(&index.list)
@@ -115,7 +160,11 @@ func (index *fileIndex) Export(path string) error {
 
 	// Close encoder prior to checking file size,
 	// to ensure the correct value is returned.
-	z.Close()
+	// If no compression is used, skip this step,
+	// as the encoder is just the file itself.
+	if Compression != "none" {
+		encoder.Close()
+	}
 
 	stats, err := file.Stat()
 	if err != nil {
@@ -149,13 +198,13 @@ func (index *fileIndex) Import(path string) error {
 		return err
 	}
 
-	z, err := zstd.NewReader(file)
+	reader, err := getReader(Compression, file)
 	if err != nil {
 		return err
 	}
-	defer z.Close()
+	defer reader.Close()
 
-	dec := gob.NewDecoder(z)
+	dec := gob.NewDecoder(reader)
 
 	index.mutex.Lock()
 	err = dec.Decode(&index.list)

cmd/root.go

@@ -8,13 +8,14 @@ import (
 	"log"
 	"math"
 	"regexp"
+	"slices"
 
 	"github.com/spf13/cobra"
 )
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "3.9.1"
+	ReleaseVersion    string = "3.10.0"
 )
 
 var (
@@ -27,6 +28,7 @@ var (
 	CaseSensitive  bool
 	Code           bool
 	CodeTheme      string
+	Compression    string
 	Concurrency    int
 	DisableButtons bool
 	ExitOnError    bool
@@ -57,6 +59,15 @@ var (
 	Version        bool
 	Videos         bool
 
+	CompressionFormats = []string{
+		"flate",
+		"gzip",
+		"lzw",
+		"none",
+		"zlib",
+		"zstd",
+	}
+
 	RequiredArgs = []string{
 		"all",
 		"audio",
@@ -84,6 +95,8 @@ var (
 				return ErrInvalidConcurrency
 			case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
 				return ErrInvalidIgnoreFile
+			case !slices.Contains(CompressionFormats, Compression):
+				return ErrInvalidCompression
 			case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
 				return ErrInvalidAdminPrefix
 			case AdminPrefix != "":
@@ -120,9 +133,10 @@ func init() {
 	rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
 	rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
 	rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
+	rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (flate, gzip, lzw, none, zlib, zstd)")
 	rootCmd.Flags().IntVar(&Concurrency, "concurrency", 8192, "maximum concurrency for scan threads")
 	rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")
-	rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing the error")
+	rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing error")
 	rootCmd.Flags().BoolVar(&Fallback, "fallback", false, "serve files as application/octet-stream if no matching format is registered")
 	rootCmd.Flags().BoolVarP(&Filtering, "filter", "f", false, "enable filtering")
 	rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)")