Re-use zstd encoder

cmd/files.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/klauspost/compress/zstd"
 	"seedno.de/seednode/roulette/types"
 )
 
@@ -447,18 +448,18 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
 	return list
 }
 
-func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
+func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) []string {
 	switch {
 	case Index && !index.isEmpty() && filters.isEmpty():
 		return index.List()
 	case Index && !index.isEmpty() && !filters.isEmpty():
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && !filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
+		index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
 
 		return filters.apply(index.List())
 	case Index && index.isEmpty() && filters.isEmpty():
-		index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
+		index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
 
 		return index.List()
 	case !Index && !filters.isEmpty():

cmd/index.go

@@ -7,7 +7,6 @@ package cmd
 import (
 	"encoding/gob"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
 	"sync"
@@ -62,7 +61,7 @@ func (index *fileIndex) remove(path string) {
 	index.mutex.Unlock()
 }
 
-func (index *fileIndex) set(val []string, errorChannel chan<- error) {
+func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel chan<- error) {
 	length := len(val)
 
 	if length < 1 {
@@ -75,7 +74,7 @@ func (index *fileIndex) set(val []string, errorChannel chan<- error) {
 	index.mutex.Unlock()
 
 	if Index && IndexFile != "" {
-		index.Export(IndexFile, errorChannel)
+		index.Export(IndexFile, encoder, errorChannel)
 	}
 }
 
@@ -93,20 +92,7 @@ func (index *fileIndex) isEmpty() bool {
 	return length == 0
 }
 
-func getReader(file io.Reader) (*zstd.Decoder, error) {
+func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel chan<- error) {
-	reader, err := zstd.NewReader(file)
-	if err != nil {
-		return nil, err
-	}
-
-	return reader, nil
-}
-
-func getWriter(file io.WriteCloser) (*zstd.Encoder, error) {
-	return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
-}
-
-func (index *fileIndex) Export(path string, errorChannel chan<- error) {
 	startTime := time.Now()
 
 	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
@@ -117,14 +103,8 @@ func (index *fileIndex) Export(path string, errorChannel chan<- error) {
 	}
 	defer file.Close()
 
-	encoder, err := getWriter(file)
+	encoder.Reset(file)
-	if err != nil {
-		errorChannel <- err
 
-		encoder.Close()
-
-		return
-	}
 	defer encoder.Close()
 
 	enc := gob.NewEncoder(encoder)
@@ -181,7 +161,7 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
 		return
 	}
 
-	reader, err := getReader(file)
+	reader, err := zstd.NewReader(file)
 	if err != nil {
 		errorChannel <- err
 
@@ -214,13 +194,13 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
 	}
 }
 
-func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
+func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		startTime := time.Now()
 
 		index.clear()
 
-		fileList(args, &filters{}, "", index, formats, errorChannel)
+		fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
 
 		w.Header().Set("Content-Type", "text/plain")
 
@@ -241,10 +221,10 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
 	}
 }
 
-func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
+func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
 	if IndexFile != "" {
 		index.Import(IndexFile, errorChannel)
 	}
 
-	fileList(args, &filters{}, "", index, formats, errorChannel)
+	fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
 }

cmd/root.go

@@ -17,7 +17,7 @@ import (
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "5.4.0"
+	ReleaseVersion    string = "5.4.1"
 )
 
 var (

cmd/web.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/julienschmidt/httprouter"
+	"github.com/klauspost/compress/zstd"
 	"github.com/yosssi/gohtml"
 	"seedno.de/seednode/roulette/types"
 	"seedno.de/seednode/roulette/types/audio"
@@ -31,6 +32,8 @@ import (
 	"seedno.de/seednode/roulette/types/video"
 )
 
+var ()
+
 const (
 	logDate            string        = `2006-01-02T15:04:05.000-07:00`
 	sourcePrefix       string        = `/source`
@@ -168,7 +171,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error
 	}
 }
 
-func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
+func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		refererUri, err := stripQueryParams(refererToUri(r.Referer()))
 		if err != nil {
@@ -203,7 +206,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChann
 			}
 		}
 
-		list := fileList(paths, filters, sortOrder, index, formats, errorChannel)
+		list := fileList(paths, filters, sortOrder, index, formats, encoder, errorChannel)
 
 	loop:
 		for timeout := time.After(timeout); ; {
@@ -576,7 +579,12 @@ func ServePage(args []string) error {
 		}
 	}()
 
-	registerHandler(mux, Prefix, serveRoot(paths, index, formats, errorChannel))
+	encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
+	if err != nil {
+		return err
+	}
+
+	registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel))
 
 	Prefix = strings.TrimSuffix(Prefix, "/")
 
@@ -595,9 +603,9 @@ func ServePage(args []string) error {
 	registerHandler(mux, Prefix+"/version", serveVersion(errorChannel))
 
 	if Index {
-		registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel))
+		registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel))
 
-		importIndex(paths, index, formats, errorChannel)
+		importIndex(paths, index, formats, encoder, errorChannel)
 	}
 
 	if Info {