Add garbage collection after every index rebuild, to test something

README.md

@@ -25,10 +25,15 @@ You can restrict access to certain functionality by prepending a secret string t
 For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`.
 
 The restricted paths are:
-- `/debug/pprof/`
+- `/debug/pprof/allocs`
+- `/debug/pprof/block`
 - `/debug/pprof/cmdline`
+- `/debug/pprof/goroutine`
+- `/debug/pprof/heap`
+- `/debug/pprof/mutex`
 - `/debug/pprof/profile`
 - `/debug/pprof/symbol`
+- `/debug/pprof/threadcreate`
 - `/debug/pprof/trace`
 - `/extensions/available`
 - `/extensions/enabled`
@@ -153,8 +158,6 @@ Flags:
       --case-sensitive        use case-sensitive matching for filters
       --code                  enable support for source code files
       --code-theme string     theme for source code syntax highlighting (default "solarized-dark256")
-      --compression string    compression format to use for index (none, zlib, zstd) (default "zstd")
-      --compression-fast      use fastest compression level (default is best)
       --concurrency int       maximum concurrency for scan threads (default 2147483647)
   -d, --debug                 display even more verbose logs
       --disable-buttons       disable first/prev/next/last buttons

cmd/errors.go

@@ -16,7 +16,6 @@ import (
 
 var (
 	ErrInvalidAdminPrefix    = errors.New("admin path must match the pattern " + AllowedCharacters)
-	ErrInvalidCompression    = errors.New("supported compression formats: none, zlib, zstd")
 	ErrInvalidConcurrency    = errors.New("concurrency limit must be a positive integer")
 	ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit")
 	ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647")

cmd/files.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/klauspost/compress/zstd"
 	"seedno.de/seednode/roulette/types"
 )
 
@@ -447,18 +448,18 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
 	return list
 }
 
-func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
+func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) []string {
 	switch {
 	case Index && !index.isEmpty() && filters.isEmpty():
 		return index.List()
 	case Index && !index.isEmpty() && !filters.isEmpty():
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && !filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel))
+		index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
 
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel))
+		index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
 
 		return index.List()
 	case !Index && !filters.isEmpty():

cmd/index.go

@@ -5,12 +5,11 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 package cmd
 
 import (
-	"compress/zlib"
 	"encoding/gob"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
+	"runtime"
 	"sync"
 	"time"
 
@@ -19,12 +18,6 @@ import (
 	"seedno.de/seednode/roulette/types"
 )
 
-var CompressionFormats = []string{
-	"none",
-	"zlib",
-	"zstd",
-}
-
 type fileIndex struct {
 	mutex *sync.RWMutex
 	list  []string
@@ -63,7 +56,7 @@ func (index *fileIndex) remove(path string) {
 	index.mutex.Unlock()
 }
 
-func (index *fileIndex) set(val []string) {
+func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel chan<- error) {
 	length := len(val)
 
 	if length < 1 {
@@ -76,7 +69,7 @@ func (index *fileIndex) set(val []string) {
 	index.mutex.Unlock()
 
 	if Index && IndexFile != "" {
-		index.Export(IndexFile)
+		index.Export(IndexFile, encoder, errorChannel)
 	}
 }
 
@@ -94,54 +87,19 @@ func (index *fileIndex) isEmpty() bool {
 	return length == 0
 }
 
-func getReader(format string, file io.Reader) (io.ReadCloser, error) {
-	switch format {
-	case "none":
-		return io.NopCloser(file), nil
-	case "zlib":
-		return zlib.NewReader(file)
-	case "zstd":
-		reader, err := zstd.NewReader(file)
-		if err != nil {
-			return io.NopCloser(file), err
-		}
-
-		return reader.IOReadCloser(), nil
-	}
-
-	return io.NopCloser(file), ErrInvalidCompression
-}
-
-func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
-	switch {
-	case format == "none":
-		return file, nil
-	case format == "zlib" && CompressionFast:
-		return zlib.NewWriterLevel(file, zlib.BestSpeed)
-	case format == "zlib":
-		return zlib.NewWriterLevel(file, zlib.BestCompression)
-	case format == "zstd" && CompressionFast:
-		return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest))
-	case format == "zstd":
-		return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
-	}
-
-	return file, ErrInvalidCompression
-}
-
-func (index *fileIndex) Export(path string) error {
+func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel chan<- error) {
 	startTime := time.Now()
 
 	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 	defer file.Close()
 
-	encoder, err := getWriter(Compression, file)
-	if err != nil {
-		return err
-	}
+	encoder.Reset(file)
+
 	defer encoder.Close()
 
 	enc := gob.NewEncoder(encoder)
@@ -151,22 +109,22 @@ func (index *fileIndex) Export(path string) error {
 	if err != nil {
 		index.mutex.RUnlock()
 
-		return err
+		errorChannel <- err
+
+		return
 	}
 	length := len(index.list)
 	index.mutex.RUnlock()
 
 	// Close encoder prior to checking file size,
 	// to ensure the correct value is returned.
-	// If no compression is used, skip this step,
-	// as the encoder is just the file itself.
-	if Compression != "none" {
-		encoder.Close()
-	}
+	encoder.Close()
 
 	stats, err := file.Stat()
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 
 	if Verbose {
@@ -178,27 +136,31 @@ func (index *fileIndex) Export(path string) error {
 			time.Since(startTime).Round(time.Microsecond),
 		)
 	}
-
-	return nil
 }
 
-func (index *fileIndex) Import(path string) error {
+func (index *fileIndex) Import(path string, errorChannel chan<- error) {
 	startTime := time.Now()
 
 	file, err := os.OpenFile(path, os.O_RDONLY, 0600)
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 	defer file.Close()
 
 	stats, err := file.Stat()
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 
-	reader, err := getReader(Compression, file)
+	reader, err := zstd.NewReader(file)
 	if err != nil {
-		return err
+		errorChannel <- err
+
+		return
 	}
 	defer reader.Close()
 
@@ -209,7 +171,9 @@ func (index *fileIndex) Import(path string) error {
 	if err != nil {
 		index.mutex.Unlock()
 
-		return err
+		errorChannel <- err
+
+		return
 	}
 	length := len(index.list)
 	index.mutex.Unlock()
@@ -223,17 +187,15 @@ func (index *fileIndex) Import(path string) error {
 			time.Since(startTime).Round(time.Microsecond),
 		)
 	}
-
-	return nil
 }
 
-func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
+func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		startTime := time.Now()
 
 		index.clear()
 
-		fileList(args, &filters{}, "", index, formats, errorChannel)
+		fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
 
 		w.Header().Set("Content-Type", "text/plain")
 
@@ -251,16 +213,15 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
 				time.Since(startTime).Round(time.Microsecond),
 			)
 		}
+
+		runtime.GC()
 	}
 }
 
-func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
+func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
 	if IndexFile != "" {
-		err := index.Import(IndexFile)
-		if err == nil {
-			return
-		}
+		index.Import(IndexFile, errorChannel)
 	}
 
-	fileList(args, &filters{}, "", index, formats, errorChannel)
+	fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
 }

cmd/profile.go

@@ -5,34 +5,20 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 package cmd
 
 import (
-	"fmt"
-	"net/http"
 	"net/http/pprof"
-	"strings"
-	"time"
 
 	"github.com/julienschmidt/httprouter"
 )
 
-func registerProfileHandler(mux *httprouter.Router, verb, path string, handler http.HandlerFunc) {
-	mux.HandlerFunc(verb, path, handler)
-
-	if Redact && AdminPrefix != "" {
-		path = strings.ReplaceAll(path, AdminPrefix, "/<admin_prefix>")
-	}
-
-	if Handlers {
-		fmt.Printf("%s | SERVE: Registered handler for %s\n",
-			time.Now().Format(logDate),
-			path,
-		)
-	}
-}
-
 func registerProfileHandlers(mux *httprouter.Router) {
-	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/", pprof.Index)
-	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/cmdline", pprof.Cmdline)
-	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/profile", pprof.Profile)
-	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/symbol", pprof.Symbol)
-	registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/trace", pprof.Trace)
+	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/allocs", pprof.Handler("allocs"))
+	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/block", pprof.Handler("block"))
+	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/goroutine", pprof.Handler("goroutine"))
+	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/heap", pprof.Handler("heap"))
+	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/mutex", pprof.Handler("mutex"))
+	mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/profile", pprof.Profile)
+	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/symbol", pprof.Symbol)
+	mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/trace", pprof.Trace)
 }

cmd/root.go

@@ -10,7 +10,6 @@ import (
 	"math"
 	"os"
 	"regexp"
-	"slices"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -18,51 +17,49 @@ import (
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "5.2.0"
+	ReleaseVersion    string = "5.4.3"
 )
 
 var (
-	AdminPrefix     string
-	All             bool
-	AllowEmpty      bool
-	Audio           bool
-	BinaryPrefix    bool
-	Bind            string
-	CaseSensitive   bool
-	Code            bool
-	CodeTheme       string
-	Compression     string
-	CompressionFast bool
-	Concurrency     int
-	Debug           bool
-	DisableButtons  bool
-	ExitOnError     bool
-	Fallback        bool
-	Filtering       bool
-	Flash           bool
-	Fun             bool
-	Handlers        bool
-	Ignore          bool
-	IgnoreFile      string
-	Images          bool
-	Index           bool
-	IndexFile       string
-	Info            bool
-	MaxFileCount    int
-	MinFileCount    int
-	PageLength      int
-	Port            int
-	Prefix          string
-	Profile         bool
-	Recursive       bool
-	Redact          bool
-	Refresh         bool
-	Russian         bool
-	Sorting         bool
-	Text            bool
-	Verbose         bool
-	Version         bool
-	Videos          bool
+	AdminPrefix    string
+	All            bool
+	AllowEmpty     bool
+	Audio          bool
+	BinaryPrefix   bool
+	Bind           string
+	CaseSensitive  bool
+	Code           bool
+	CodeTheme      string
+	Concurrency    int
+	Debug          bool
+	DisableButtons bool
+	ExitOnError    bool
+	Fallback       bool
+	Filtering      bool
+	Flash          bool
+	Fun            bool
+	Handlers       bool
+	Ignore         bool
+	IgnoreFile     string
+	Images         bool
+	Index          bool
+	IndexFile      string
+	Info           bool
+	MaxFileCount   int
+	MinFileCount   int
+	PageLength     int
+	Port           int
+	Prefix         string
+	Profile        bool
+	Recursive      bool
+	Redact         bool
+	Refresh        bool
+	Russian        bool
+	Sorting        bool
+	Text           bool
+	Verbose        bool
+	Version        bool
+	Videos         bool
 
 	RequiredArgs = []string{
 		"all",
@@ -91,8 +88,6 @@ var (
 				return ErrInvalidConcurrency
 			case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
 				return ErrInvalidIgnoreFile
-			case !slices.Contains(CompressionFormats, Compression):
-				return ErrInvalidCompression
 			case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
 				return ErrInvalidAdminPrefix
 			case AdminPrefix != "":
@@ -131,8 +126,6 @@ func init() {
 	rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
 	rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
 	rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
-	rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (none, zlib, zstd)")
-	rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
 	rootCmd.Flags().IntVar(&Concurrency, "concurrency", 10240, "maximum concurrency for scan threads")
 	rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
 	rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")

cmd/web.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/julienschmidt/httprouter"
+	"github.com/klauspost/compress/zstd"
 	"github.com/yosssi/gohtml"
 	"seedno.de/seednode/roulette/types"
 	"seedno.de/seednode/roulette/types/audio"
@@ -31,6 +32,8 @@ import (
 	"seedno.de/seednode/roulette/types/video"
 )
 
+var ()
+
 const (
 	logDate            string        = `2006-01-02T15:04:05.000-07:00`
 	sourcePrefix       string        = `/source`
@@ -168,7 +171,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error
 	}
 }
 
-func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
+func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		refererUri, err := stripQueryParams(refererToUri(r.Referer()))
 		if err != nil {
@@ -203,7 +206,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChann
 			}
 		}
 
-		list := fileList(paths, filters, sortOrder, index, formats, errorChannel)
+		list := fileList(paths, filters, sortOrder, index, formats, encoder, errorChannel)
 
 	loop:
 		for timeout := time.After(timeout); ; {
@@ -576,7 +579,12 @@ func ServePage(args []string) error {
 		}
 	}()
 
-	registerHandler(mux, Prefix, serveRoot(paths, index, formats, errorChannel))
+	encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
+	if err != nil {
+		return err
+	}
+
+	registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel))
 
 	Prefix = strings.TrimSuffix(Prefix, "/")
 
@@ -595,9 +603,9 @@ func ServePage(args []string) error {
 	registerHandler(mux, Prefix+"/version", serveVersion(errorChannel))
 
 	if Index {
-		registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel))
+		registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel))
 
-		importIndex(paths, index, formats, errorChannel)
+		importIndex(paths, index, formats, encoder, errorChannel)
 	}
 
 	if Info {