SeanOMik
/
roulette
mirror of https://git.seedno.de/seednode/roulette.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Re-use zstd encoder

This commit is contained in:
Seednode 2024-01-14 09:43:22 -06:00
parent 2f06ae3605
commit 6bd97f30c2
4 changed files with 27 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/files.go
View File

@ -21,6 +21,7 @@ import (
"sync"
"time"
"github.com/klauspost/compress/zstd"
"seedno.de/seednode/roulette/types"
)
@ -447,18 +448,18 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
return list
}
func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) []string {
switch {
case Index && !index.isEmpty() && filters.isEmpty():
return index.List()
case Index && !index.isEmpty() && !filters.isEmpty():
return filters.apply(index.List())
case Index && index.isEmpty() && !filters.isEmpty():
index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
return filters.apply(index.List())
case Index && index.isEmpty() && filters.isEmpty():
index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
return index.List()
case !Index && !filters.isEmpty():

38
cmd/index.go
View File

@ -7,7 +7,6 @@ package cmd
import (
"encoding/gob"
"fmt"
"io"
"net/http"
"os"
"sync"
@ -62,7 +61,7 @@ func (index *fileIndex) remove(path string) {
index.mutex.Unlock()
}
func (index *fileIndex) set(val []string, errorChannel chan<- error) {
func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel chan<- error) {
length := len(val)
if length < 1 {
@ -75,7 +74,7 @@ func (index *fileIndex) set(val []string, errorChannel chan<- error) {
index.mutex.Unlock()
if Index && IndexFile != "" {
index.Export(IndexFile, errorChannel)
index.Export(IndexFile, encoder, errorChannel)
}
}
@ -93,20 +92,7 @@ func (index *fileIndex) isEmpty() bool {
return length == 0
}
func getReader(file io.Reader) (*zstd.Decoder, error) {
reader, err := zstd.NewReader(file)
if err != nil {
return nil, err
}
return reader, nil
}
func getWriter(file io.WriteCloser) (*zstd.Encoder, error) {
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
}
func (index *fileIndex) Export(path string, errorChannel chan<- error) {
func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel chan<- error) {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
@ -117,14 +103,8 @@ func (index *fileIndex) Export(path string, errorChannel chan<- error) {
}
defer file.Close()
encoder, err := getWriter(file)
if err != nil {
errorChannel <- err
encoder.Reset(file)
encoder.Close()
return
}
defer encoder.Close()
enc := gob.NewEncoder(encoder)
@ -181,7 +161,7 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
return
}
reader, err := getReader(file)
reader, err := zstd.NewReader(file)
if err != nil {
errorChannel <- err
@ -214,13 +194,13 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
}
}
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
index.clear()
fileList(args, &filters{}, "", index, formats, errorChannel)
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
w.Header().Set("Content-Type", "text/plain")
@ -241,10 +221,10 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
}
}
func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
if IndexFile != "" {
index.Import(IndexFile, errorChannel)
}
fileList(args, &filters{}, "", index, formats, errorChannel)
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
}

2
cmd/root.go
View File

@ -17,7 +17,7 @@ import (
const (
AllowedCharacters string = `^[A-z0-9.\-_]+$`
ReleaseVersion string = "5.4.0"
ReleaseVersion string = "5.4.1"
)
var (

18
cmd/web.go
View File

@ -21,6 +21,7 @@ import (
"time"
"github.com/julienschmidt/httprouter"
"github.com/klauspost/compress/zstd"
"github.com/yosssi/gohtml"
"seedno.de/seednode/roulette/types"
"seedno.de/seednode/roulette/types/audio"
@ -31,6 +32,8 @@ import (
"seedno.de/seednode/roulette/types/video"
)
var ()
const (
logDate string = `2006-01-02T15:04:05.000-07:00`
sourcePrefix string = `/source`
@ -168,7 +171,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error
}
}
func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
refererUri, err := stripQueryParams(refererToUri(r.Referer()))
if err != nil {
@ -203,7 +206,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChann
}
}
list := fileList(paths, filters, sortOrder, index, formats, errorChannel)
list := fileList(paths, filters, sortOrder, index, formats, encoder, errorChannel)
loop:
for timeout := time.After(timeout); ; {
@ -576,7 +579,12 @@ func ServePage(args []string) error {
}
}()
registerHandler(mux, Prefix, serveRoot(paths, index, formats, errorChannel))
encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
if err != nil {
return err
}
registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel))
Prefix = strings.TrimSuffix(Prefix, "/")
@ -595,9 +603,9 @@ func ServePage(args []string) error {
registerHandler(mux, Prefix+"/version", serveVersion(errorChannel))
if Index {
registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel))
importIndex(paths, index, formats, errorChannel)
importIndex(paths, index, formats, encoder, errorChannel)
}
if Info {