diff --git a/cmd/files.go b/cmd/files.go index 605cbda..8a19dbb 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/klauspost/compress/zstd" "seedno.de/seednode/roulette/types" ) @@ -447,18 +448,18 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type return list } -func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string { +func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) []string { switch { case Index && !index.isEmpty() && filters.isEmpty(): return index.List() case Index && !index.isEmpty() && !filters.isEmpty(): return filters.apply(index.List()) case Index && index.isEmpty() && !filters.isEmpty(): - index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel) + index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel) return filters.apply(index.List()) case Index && index.isEmpty() && filters.isEmpty(): - index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel) + index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel) return index.List() case !Index && !filters.isEmpty(): diff --git a/cmd/index.go b/cmd/index.go index 6583e76..1bc6c1d 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -7,7 +7,6 @@ package cmd import ( "encoding/gob" "fmt" - "io" "net/http" "os" "sync" @@ -62,7 +61,7 @@ func (index *fileIndex) remove(path string) { index.mutex.Unlock() } -func (index *fileIndex) set(val []string, errorChannel chan<- error) { +func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel chan<- error) { length := len(val) if length < 1 { @@ -75,7 +74,7 @@ func (index *fileIndex) set(val []string, errorChannel chan<- error) { index.mutex.Unlock() if Index && IndexFile != "" { - index.Export(IndexFile, errorChannel) + index.Export(IndexFile, encoder, errorChannel) } } @@ -93,20 +92,7 @@ func (index *fileIndex) isEmpty() bool { return length == 0 } -func getReader(file io.Reader) (*zstd.Decoder, error) { - reader, err := zstd.NewReader(file) - if err != nil { - return nil, err - } - - return reader, nil -} - -func getWriter(file io.WriteCloser) (*zstd.Encoder, error) { - return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) -} - -func (index *fileIndex) Export(path string, errorChannel chan<- error) { +func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel chan<- error) { startTime := time.Now() file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) @@ -117,14 +103,8 @@ func (index *fileIndex) Export(path string, errorChannel chan<- error) { } defer file.Close() - encoder, err := getWriter(file) - if err != nil { - errorChannel <- err + encoder.Reset(file) - encoder.Close() - - return - } defer encoder.Close() enc := gob.NewEncoder(encoder) @@ -181,7 +161,7 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) { return } - reader, err := getReader(file) + reader, err := zstd.NewReader(file) if err != nil { errorChannel <- err @@ -214,13 +194,13 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) { } } -func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle { +func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { startTime := time.Now() index.clear() - fileList(args, &filters{}, "", index, formats, errorChannel) + fileList(args, &filters{}, "", index, formats, encoder, errorChannel) w.Header().Set("Content-Type", "text/plain") @@ -241,10 +221,10 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err } } -func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { +func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) { if IndexFile != "" { index.Import(IndexFile, errorChannel) } - fileList(args, &filters{}, "", index, formats, errorChannel) + fileList(args, &filters{}, "", index, formats, encoder, errorChannel) } diff --git a/cmd/root.go b/cmd/root.go index e8bbe0c..cb669d8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( const ( AllowedCharacters string = `^[A-z0-9.\-_]+$` - ReleaseVersion string = "5.4.0" + ReleaseVersion string = "5.4.1" ) var ( diff --git a/cmd/web.go b/cmd/web.go index 4400318..23bfbfc 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -21,6 +21,7 @@ import ( "time" "github.com/julienschmidt/httprouter" + "github.com/klauspost/compress/zstd" "github.com/yosssi/gohtml" "seedno.de/seednode/roulette/types" "seedno.de/seednode/roulette/types/audio" @@ -31,6 +32,8 @@ import ( "seedno.de/seednode/roulette/types/video" ) +var () + const ( logDate string = `2006-01-02T15:04:05.000-07:00` sourcePrefix string = `/source` @@ -168,7 +171,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error } } -func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle { +func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { refererUri, err := stripQueryParams(refererToUri(r.Referer())) if err != nil { @@ -203,7 +206,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChann } } - list := fileList(paths, filters, sortOrder, index, formats, errorChannel) + list := fileList(paths, filters, sortOrder, index, formats, encoder, errorChannel) loop: for timeout := time.After(timeout); ; { @@ -576,7 +579,12 @@ func ServePage(args []string) error { } }() - registerHandler(mux, Prefix, serveRoot(paths, index, formats, errorChannel)) + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) + if err != nil { + return err + } + + registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel)) Prefix = strings.TrimSuffix(Prefix, "/") @@ -595,9 +603,9 @@ func ServePage(args []string) error { registerHandler(mux, Prefix+"/version", serveVersion(errorChannel)) if Index { - registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel)) - importIndex(paths, index, formats, errorChannel) + importIndex(paths, index, formats, encoder, errorChannel) } if Info {