README.md

@@ -25,15 +25,10 @@ You can restrict access to certain functionality by prepending a secret string t
 For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`.
 
 The restricted paths are:
-- `/debug/pprof/allocs`
+- `/debug/pprof/`
-- `/debug/pprof/block`
 - `/debug/pprof/cmdline`
-- `/debug/pprof/goroutine`
-- `/debug/pprof/heap`
-- `/debug/pprof/mutex`
 - `/debug/pprof/profile`
 - `/debug/pprof/symbol`
-- `/debug/pprof/threadcreate`
 - `/debug/pprof/trace`
 - `/extensions/available`
 - `/extensions/enabled`
@@ -158,6 +153,8 @@ Flags:
       --case-sensitive        use case-sensitive matching for filters
       --code                  enable support for source code files
       --code-theme string     theme for source code syntax highlighting (default "solarized-dark256")
+      --compression string    compression format to use for index (none, zlib, zstd) (default "zstd")
+      --compression-fast      use fastest compression level (default is best)
       --concurrency int       maximum concurrency for scan threads (default 2147483647)
   -d, --debug                 display even more verbose logs
       --disable-buttons       disable first/prev/next/last buttons

cmd/errors.go

@@ -16,6 +16,7 @@ import (
 
 var (
 	ErrInvalidAdminPrefix    = errors.New("admin path must match the pattern " + AllowedCharacters)
+	ErrInvalidCompression    = errors.New("supported compression formats: none, zlib, zstd")
 	ErrInvalidConcurrency    = errors.New("concurrency limit must be a positive integer")
 	ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit")
 	ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647")

cmd/files.go

@@ -21,7 +21,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/klauspost/compress/zstd"
 	"seedno.de/seednode/roulette/types"
 )
 
@@ -448,18 +447,18 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
 	return list
 }
 
-func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) []string {
+func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
 	switch {
 	case Index && !index.isEmpty() && filters.isEmpty():
 		return index.List()
 	case Index && !index.isEmpty() && !filters.isEmpty():
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && !filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
+		index.set(scanPaths(paths, sort, index, formats, errorChannel))
 
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
+		index.set(scanPaths(paths, sort, index, formats, errorChannel))
 
 		return index.List()
 	case !Index && !filters.isEmpty():

cmd/index.go

@@ -5,11 +5,12 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 package cmd
 
 import (
+	"compress/zlib"
 	"encoding/gob"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
-	"runtime"
 	"sync"
 	"time"
 
@@ -18,6 +19,12 @@ import (
 	"seedno.de/seednode/roulette/types"
 )
 
+var CompressionFormats = []string{
+	"none",
+	"zlib",
+	"zstd",
+}
+
 type fileIndex struct {
 	mutex *sync.RWMutex
 	list  []string
@@ -56,7 +63,7 @@ func (index *fileIndex) remove(path string) {
 	index.mutex.Unlock()
 }
 
-func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel chan<- error) {
+func (index *fileIndex) set(val []string) {
 	length := len(val)
 
 	if length < 1 {
@@ -69,7 +76,7 @@ func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel ch
 	index.mutex.Unlock()
 
 	if Index && IndexFile != "" {
-		index.Export(IndexFile, encoder, errorChannel)
+		index.Export(IndexFile)
 	}
 }
 
@@ -87,19 +94,54 @@ func (index *fileIndex) isEmpty() bool {
 	return length == 0
 }
 
-func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel chan<- error) {
+func getReader(format string, file io.Reader) (io.ReadCloser, error) {
+	switch format {
+	case "none":
+		return io.NopCloser(file), nil
+	case "zlib":
+		return zlib.NewReader(file)
+	case "zstd":
+		reader, err := zstd.NewReader(file)
+		if err != nil {
+			return io.NopCloser(file), err
+		}
+
+		return reader.IOReadCloser(), nil
+	}
+
+	return io.NopCloser(file), ErrInvalidCompression
+}
+
+func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
+	switch {
+	case format == "none":
+		return file, nil
+	case format == "zlib" && CompressionFast:
+		return zlib.NewWriterLevel(file, zlib.BestSpeed)
+	case format == "zlib":
+		return zlib.NewWriterLevel(file, zlib.BestCompression)
+	case format == "zstd" && CompressionFast:
+		return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest))
+	case format == "zstd":
+		return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
+	}
+
+	return file, ErrInvalidCompression
+}
+
+func (index *fileIndex) Export(path string) error {
 	startTime := time.Now()
 
 	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		errorChannel <- err
+		return err
-
-		return
 	}
 	defer file.Close()
 
-	encoder.Reset(file)
+	encoder, err := getWriter(Compression, file)
-
+	if err != nil {
+		return err
+	}
 	defer encoder.Close()
 
 	enc := gob.NewEncoder(encoder)
@@ -109,22 +151,22 @@ func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel
 	if err != nil {
 		index.mutex.RUnlock()
 
-		errorChannel <- err
+		return err
-
-		return
 	}
 	length := len(index.list)
 	index.mutex.RUnlock()
 
 	// Close encoder prior to checking file size,
 	// to ensure the correct value is returned.
+	// If no compression is used, skip this step,
+	// as the encoder is just the file itself.
+	if Compression != "none" {
 		encoder.Close()
+	}
 
 	stats, err := file.Stat()
 	if err != nil {
-		errorChannel <- err
+		return err
-
-		return
 	}
 
 	if Verbose {
@@ -136,31 +178,27 @@ func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel
 			time.Since(startTime).Round(time.Microsecond),
 		)
 	}
+
+	return nil
 }
 
-func (index *fileIndex) Import(path string, errorChannel chan<- error) {
+func (index *fileIndex) Import(path string) error {
 	startTime := time.Now()
 
 	file, err := os.OpenFile(path, os.O_RDONLY, 0600)
 	if err != nil {
-		errorChannel <- err
+		return err
-
-		return
 	}
 	defer file.Close()
 
 	stats, err := file.Stat()
 	if err != nil {
-		errorChannel <- err
+		return err
-
-		return
 	}
 
-	reader, err := zstd.NewReader(file)
+	reader, err := getReader(Compression, file)
 	if err != nil {
-		errorChannel <- err
+		return err
-
-		return
 	}
 	defer reader.Close()
 
@@ -171,9 +209,7 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
 	if err != nil {
 		index.mutex.Unlock()
 
-		errorChannel <- err
+		return err
-
-		return
 	}
 	length := len(index.list)
 	index.mutex.Unlock()
@@ -187,15 +223,17 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
 			time.Since(startTime).Round(time.Microsecond),
 		)
 	}
+
+	return nil
 }
 
-func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
+func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		startTime := time.Now()
 
 		index.clear()
 
-		fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
+		fileList(args, &filters{}, "", index, formats, errorChannel)
 
 		w.Header().Set("Content-Type", "text/plain")
 
@@ -213,15 +251,16 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, enc
 				time.Since(startTime).Round(time.Microsecond),
 			)
 		}
-
-		runtime.GC()
 	}
 }
 
-func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
+func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
 	if IndexFile != "" {
-		index.Import(IndexFile, errorChannel)
+		err := index.Import(IndexFile)
+		if err == nil {
+			return
+		}
 	}
 
-	fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
+	fileList(args, &filters{}, "", index, formats, errorChannel)
 }

cmd/profile.go

@@ -5,20 +5,34 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 package cmd
 
 import (
+	"fmt"
+	"net/http"
 	"net/http/pprof"
+	"strings"
+	"time"
 
 	"github.com/julienschmidt/httprouter"
 )
 
-func registerProfileHandlers(mux *httprouter.Router) {
+func registerProfileHandler(mux *httprouter.Router, verb, path string, handler http.HandlerFunc) {
-	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/allocs", pprof.Handler("allocs"))
+	mux.HandlerFunc(verb, path, handler)
-	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/block", pprof.Handler("block"))
+
-	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/goroutine", pprof.Handler("goroutine"))
+	if Redact && AdminPrefix != "" {
-	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/heap", pprof.Handler("heap"))
+		path = strings.ReplaceAll(path, AdminPrefix, "/<admin_prefix>")
-	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/mutex", pprof.Handler("mutex"))
+	}
-	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+
-	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/cmdline", pprof.Cmdline)
+	if Handlers {
-	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/profile", pprof.Profile)
+		fmt.Printf("%s | SERVE: Registered handler for %s\n",
-	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/symbol", pprof.Symbol)
+			time.Now().Format(logDate),
-	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/trace", pprof.Trace)
+			path,
+		)
+	}
+}
+
+func registerProfileHandlers(mux *httprouter.Router) {
+	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/", pprof.Index)
+	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/cmdline", pprof.Cmdline)
+	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/profile", pprof.Profile)
+	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/symbol", pprof.Symbol)
+	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/trace", pprof.Trace)
 }

cmd/root.go

@@ -10,6 +10,7 @@ import (
 	"math"
 	"os"
 	"regexp"
+	"slices"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -17,7 +18,7 @@ import (
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "5.4.3"
+	ReleaseVersion    string = "5.2.0"
 )
 
 var (
@@ -30,6 +31,8 @@ var (
 	CaseSensitive   bool
 	Code            bool
 	CodeTheme       string
+	Compression     string
+	CompressionFast bool
 	Concurrency     int
 	Debug           bool
 	DisableButtons  bool
@@ -88,6 +91,8 @@ var (
 				return ErrInvalidConcurrency
 			case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
 				return ErrInvalidIgnoreFile
+			case !slices.Contains(CompressionFormats, Compression):
+				return ErrInvalidCompression
 			case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
 				return ErrInvalidAdminPrefix
 			case AdminPrefix != "":
@@ -126,6 +131,8 @@ func init() {
 	rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
 	rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
 	rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
+	rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (none, zlib, zstd)")
+	rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
 	rootCmd.Flags().IntVar(&Concurrency, "concurrency", 10240, "maximum concurrency for scan threads")
 	rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
 	rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")

cmd/web.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/julienschmidt/httprouter"
-	"github.com/klauspost/compress/zstd"
 	"github.com/yosssi/gohtml"
 	"seedno.de/seednode/roulette/types"
 	"seedno.de/seednode/roulette/types/audio"
@@ -32,8 +31,6 @@ import (
 	"seedno.de/seednode/roulette/types/video"
 )
 
-var ()
-
 const (
 	logDate            string        = `2006-01-02T15:04:05.000-07:00`
 	sourcePrefix       string        = `/source`
@@ -171,7 +168,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error
 	}
 }
 
-func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
+func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		refererUri, err := stripQueryParams(refererToUri(r.Referer()))
 		if err != nil {
@@ -206,7 +203,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z
 			}
 		}
 
-		list := fileList(paths, filters, sortOrder, index, formats, encoder, errorChannel)
+		list := fileList(paths, filters, sortOrder, index, formats, errorChannel)
 
 	loop:
 		for timeout := time.After(timeout); ; {
@@ -579,12 +576,7 @@ func ServePage(args []string) error {
 		}
 	}()
 
-	encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
+	registerHandler(mux, Prefix, serveRoot(paths, index, formats, errorChannel))
-	if err != nil {
-		return err
-	}
-
-	registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel))
 
 	Prefix = strings.TrimSuffix(Prefix, "/")
 
@@ -603,9 +595,9 @@ func ServePage(args []string) error {
 	registerHandler(mux, Prefix+"/version", serveVersion(errorChannel))
 
 	if Index {
-		registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel))
+		registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel))
 
-		importIndex(paths, index, formats, encoder, errorChannel)
+		importIndex(paths, index, formats, errorChannel)
 	}
 
 	if Info {