From 8939f00653f011a928c483e4734e29fd56529936 Mon Sep 17 00:00:00 2001 From: Seednode Date: Tue, 6 Feb 2024 10:12:55 -0600 Subject: [PATCH] Remove zstd encoder re-use, as it runs at fairly long intervals and isn't worth keeping the object alive --- cmd/files.go | 7 +++---- cmd/index.go | 30 +++++++++++++++++------------- cmd/info.go | 5 ++--- cmd/root.go | 2 +- cmd/web.go | 18 ++++++------------ 5 files changed, 29 insertions(+), 33 deletions(-) diff --git a/cmd/files.go b/cmd/files.go index d8e30a4..91c64d4 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -21,7 +21,6 @@ import ( "sync" "time" - "github.com/klauspost/compress/zstd" "seedno.de/seednode/roulette/types" ) @@ -434,18 +433,18 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type return list } -func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) []string { +func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string { switch { case Index && !index.isEmpty() && filters.isEmpty(): return index.List() case Index && !index.isEmpty() && !filters.isEmpty(): return filters.apply(index.List()) case Index && index.isEmpty() && !filters.isEmpty(): - index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel) + index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel) return filters.apply(index.List()) case Index && index.isEmpty() && filters.isEmpty(): - index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel) + index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel) return index.List() case !Index && !filters.isEmpty(): diff --git a/cmd/index.go b/cmd/index.go index 412d4d5..7a07ba2 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -58,7 +58,7 @@ func (index *fileIndex) remove(path string) { index.mutex.Unlock() } -func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel chan<- error) { +func (index *fileIndex) set(val []string, errorChannel chan<- error) { length := len(val) if length < 1 { @@ -71,7 +71,7 @@ func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel ch index.mutex.Unlock() if Index && IndexFile != "" { - index.Export(IndexFile, encoder, errorChannel) + index.Export(IndexFile, errorChannel) } } @@ -89,7 +89,7 @@ func (index *fileIndex) isEmpty() bool { return length == 0 } -func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel chan<- error) { +func (index *fileIndex) Export(path string, errorChannel chan<- error) { startTime := time.Now() file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) @@ -100,11 +100,15 @@ func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel } defer file.Close() - encoder.Reset(file) + encoder, err := zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) + if err != nil { + errorChannel <- err + return + } defer encoder.Close() - enc := gob.NewEncoder(encoder) + enc := gob.NewEncoder(file) index.mutex.RLock() err = enc.Encode(&index.list) @@ -191,18 +195,18 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) { } } -func rebuildIndex(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) { +func rebuildIndex(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { index.clear() - fileList(paths, &filters{}, "", index, formats, encoder, errorChannel) + fileList(paths, &filters{}, "", index, formats, errorChannel) } -func importIndex(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) { +func importIndex(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { if IndexFile != "" { index.Import(IndexFile, errorChannel) } - fileList(paths, &filters{}, "", index, formats, encoder, errorChannel) + fileList(paths, &filters{}, "", index, formats, errorChannel) } func serveIndex(index *fileIndex, errorChannel chan<- error) httprouter.Handle { @@ -246,7 +250,7 @@ func serveIndex(index *fileIndex, errorChannel chan<- error) httprouter.Handle { } } -func serveIndexRebuild(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { +func serveIndexRebuild(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { if Verbose { fmt.Printf("%s | SERVE: Index rebuild requested by %s\n", @@ -258,7 +262,7 @@ func serveIndexRebuild(paths []string, index *fileIndex, formats types.Types, en w.Header().Set("Content-Type", "text/plain;charset=UTF-8") - rebuildIndex(paths, index, formats, encoder, errorChannel) + rebuildIndex(paths, index, formats, errorChannel) _, err := w.Write([]byte("Ok\n")) if err != nil { @@ -269,7 +273,7 @@ func serveIndexRebuild(paths []string, index *fileIndex, formats types.Types, en } } -func registerIndexInterval(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, quit <-chan struct{}, errorChannel chan<- error) { +func registerIndexInterval(paths []string, index *fileIndex, formats types.Types, quit <-chan struct{}, errorChannel chan<- error) { interval, err := time.ParseDuration(IndexInterval) if err != nil { errorChannel <- err @@ -287,7 +291,7 @@ func registerIndexInterval(paths []string, index *fileIndex, formats types.Types fmt.Printf("%s | INDEX: Started scheduled index rebuild\n", time.Now().Format(logDate)) } - rebuildIndex(paths, index, formats, encoder, errorChannel) + rebuildIndex(paths, index, formats, errorChannel) case <-quit: ticker.Stop() diff --git a/cmd/info.go b/cmd/info.go index 426f295..e8b344d 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -10,7 +10,6 @@ import ( "time" "github.com/julienschmidt/httprouter" - "github.com/klauspost/compress/zstd" "seedno.de/seednode/roulette/types" ) @@ -76,10 +75,10 @@ func serveMediaTypes(formats types.Types, available bool, errorChannel chan<- er } } -func registerAPIHandlers(mux *httprouter.Router, paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) { +func registerAPIHandlers(mux *httprouter.Router, paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { if Index { mux.GET(Prefix+AdminPrefix+"/index", serveIndex(index, errorChannel)) - mux.POST(Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(paths, index, formats, encoder, errorChannel)) + mux.POST(Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(paths, index, formats, errorChannel)) } mux.GET(Prefix+AdminPrefix+"/extensions/available", serveExtensions(formats, true, errorChannel)) diff --git a/cmd/root.go b/cmd/root.go index 6eea5b9..f124db4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( const ( AllowedCharacters string = `^[A-z0-9.\-_]+$` - ReleaseVersion string = "8.4.2" + ReleaseVersion string = "8.4.3" ) var ( diff --git a/cmd/web.go b/cmd/web.go index 706e85f..c760489 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -21,7 +21,6 @@ import ( "time" "github.com/julienschmidt/httprouter" - "github.com/klauspost/compress/zstd" "github.com/yosssi/gohtml" "seedno.de/seednode/roulette/types" "seedno.de/seednode/roulette/types/audio" @@ -157,7 +156,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error } } -func serveRoot(paths []string, index *fileIndex, filename *regexp.Regexp, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { +func serveRoot(paths []string, index *fileIndex, filename *regexp.Regexp, formats types.Types, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Security-Policy", "default-src 'self';") @@ -194,7 +193,7 @@ func serveRoot(paths []string, index *fileIndex, filename *regexp.Regexp, format } } - list := fileList(paths, filters, sortOrder, index, formats, encoder, errorChannel) + list := fileList(paths, filters, sortOrder, index, formats, errorChannel) loop: for timeout := time.After(timeout); ; { @@ -561,18 +560,13 @@ func ServePage(args []string) error { } }() - encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) - if err != nil { - return err - } - filename := regexp.MustCompile(`(.+?)([0-9]*)(\..+)`) if !strings.HasSuffix(Prefix, "/") { Prefix = Prefix + "/" } - mux.GET(Prefix, serveRoot(paths, index, filename, formats, encoder, errorChannel)) + mux.GET(Prefix, serveRoot(paths, index, filename, formats, errorChannel)) Prefix = strings.TrimSuffix(Prefix, "/") @@ -594,14 +588,14 @@ func ServePage(args []string) error { defer close(quit) if API { - registerAPIHandlers(mux, paths, index, formats, encoder, errorChannel) + registerAPIHandlers(mux, paths, index, formats, errorChannel) } if Index { - importIndex(paths, index, formats, encoder, errorChannel) + importIndex(paths, index, formats, errorChannel) if IndexInterval != "" { - registerIndexInterval(paths, index, formats, encoder, quit, errorChannel) + registerIndexInterval(paths, index, formats, quit, errorChannel) } }