From fac4412c5c4f8e72c5524360a08a3b00c59cc7c9 Mon Sep 17 00:00:00 2001 From: Seednode Date: Fri, 5 Jan 2024 20:39:05 -0600 Subject: [PATCH] Add --compression, to allow specifying compression format for index file --- README.md | 3 ++- cmd/errors.go | 1 + cmd/index.go | 63 +++++++++++++++++++++++++++++++++++++++++++++------ cmd/root.go | 18 +++++++++++++-- 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bedd5a8..d5feaa3 100644 --- a/README.md +++ b/README.md @@ -145,9 +145,10 @@ Flags: --case-sensitive use case-sensitive matching for filters --code enable support for source code files --code-theme string theme for source code syntax highlighting (default "solarized-dark256") + --compression string compression format to use for index (flate, gzip, lzw, none, zlib, zstd) (default "zstd") --concurrency int maximum concurrency for scan threads (default 8192) --disable-buttons disable first/prev/next/last buttons - --exit-on-error shut down webserver on error, instead of just printing the error + --exit-on-error shut down webserver on error, instead of just printing error --fallback serve files as application/octet-stream if no matching format is registered -f, --filter enable filtering --flash enable support for shockwave flash files (via ruffle.rs) diff --git a/cmd/errors.go b/cmd/errors.go index 4bf4477..3fc65bf 100644 --- a/cmd/errors.go +++ b/cmd/errors.go @@ -16,6 +16,7 @@ import ( var ( ErrInvalidAdminPrefix = errors.New("admin path must match the pattern " + AllowedCharacters) + ErrInvalidCompression = errors.New("supported compression formats: flate, gzip, lzw, none, zlib, zstd") ErrInvalidConcurrency = errors.New("concurrency limit must be a positive integer") ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit") ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647") diff --git a/cmd/index.go b/cmd/index.go index efb3e4c..925aa90 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -5,8 +5,13 @@ Copyright © 2023 Seednode package cmd import ( + "compress/flate" + "compress/gzip" + "compress/lzw" + "compress/zlib" "encoding/gob" "fmt" + "io" "net/http" "os" "sync" @@ -86,6 +91,46 @@ func (index *fileIndex) isEmpty() bool { return length == 0 } +func getReader(format string, file io.Reader) (io.ReadCloser, error) { + switch format { + case "flate": + return flate.NewReader(file), nil + case "gzip": + return gzip.NewReader(file) + case "lzw": + return lzw.NewReader(file, lzw.LSB, 8), nil + case "none": + return io.NopCloser(file), nil + case "zlib": + return zlib.NewReader(file) + case "zstd": + decoder, err := zstd.NewReader(file) + + return decoder.IOReadCloser(), err + } + + return io.NopCloser(file), ErrInvalidCompression +} + +func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) { + switch format { + case "flate": + return flate.NewWriter(file, flate.DefaultCompression) + case "gzip": + return gzip.NewWriter(file), nil + case "lzw": + return lzw.NewWriter(file, lzw.LSB, 8), nil + case "none": + return file, nil + case "zlib": + return zlib.NewWriter(file), nil + case "zstd": + return zstd.NewWriter(file) + } + + return file, ErrInvalidCompression +} + func (index *fileIndex) Export(path string) error { startTime := time.Now() @@ -95,13 +140,13 @@ func (index *fileIndex) Export(path string) error { } defer file.Close() - z, err := zstd.NewWriter(file) + encoder, err := getWriter(Compression, file) if err != nil { return err } - defer z.Close() + defer encoder.Close() - enc := gob.NewEncoder(z) + enc := gob.NewEncoder(encoder) index.mutex.RLock() err = enc.Encode(&index.list) @@ -115,7 +160,11 @@ func (index *fileIndex) Export(path string) error { // Close encoder prior to checking file size, // to ensure the correct value is returned. - z.Close() + // If no compression is used, skip this step, + // as the encoder is just the file itself. + if Compression != "none" { + encoder.Close() + } stats, err := file.Stat() if err != nil { @@ -149,13 +198,13 @@ func (index *fileIndex) Import(path string) error { return err } - z, err := zstd.NewReader(file) + reader, err := getReader(Compression, file) if err != nil { return err } - defer z.Close() + defer reader.Close() - dec := gob.NewDecoder(z) + dec := gob.NewDecoder(reader) index.mutex.Lock() err = dec.Decode(&index.list) diff --git a/cmd/root.go b/cmd/root.go index 5afbc9c..5f0987b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,13 +8,14 @@ import ( "log" "math" "regexp" + "slices" "github.com/spf13/cobra" ) const ( AllowedCharacters string = `^[A-z0-9.\-_]+$` - ReleaseVersion string = "3.9.1" + ReleaseVersion string = "3.10.0" ) var ( @@ -27,6 +28,7 @@ var ( CaseSensitive bool Code bool CodeTheme string + Compression string Concurrency int DisableButtons bool ExitOnError bool @@ -57,6 +59,15 @@ var ( Version bool Videos bool + CompressionFormats = []string{ + "flate", + "gzip", + "lzw", + "none", + "zlib", + "zstd", + } + RequiredArgs = []string{ "all", "audio", @@ -84,6 +95,8 @@ var ( return ErrInvalidConcurrency case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile): return ErrInvalidIgnoreFile + case !slices.Contains(CompressionFormats, Compression): + return ErrInvalidCompression case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix): return ErrInvalidAdminPrefix case AdminPrefix != "": @@ -120,9 +133,10 @@ func init() { rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters") rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files") rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting") + rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (flate, gzip, lzw, none, zlib, zstd)") rootCmd.Flags().IntVar(&Concurrency, "concurrency", 8192, "maximum concurrency for scan threads") rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons") - rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing the error") + rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing error") rootCmd.Flags().BoolVar(&Fallback, "fallback", false, "serve files as application/octet-stream if no matching format is registered") rootCmd.Flags().BoolVarP(&Filtering, "filter", "f", false, "enable filtering") rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)")