SeanOMik
/
roulette
mirror of https://git.seedno.de/seednode/roulette.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove non-zstd compression options

This commit is contained in:
Seednode 2024-01-14 09:29:22 -06:00
parent 7868f0d7ad
commit 2f06ae3605
5 changed files with 89 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -25,10 +25,15 @@ You can restrict access to certain functionality by prepending a secret string t
For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`.
The restricted paths are:
- `/debug/pprof/`
- `/debug/pprof/allocs`
- `/debug/pprof/block`
- `/debug/pprof/cmdline`
- `/debug/pprof/goroutine`
- `/debug/pprof/heap`
- `/debug/pprof/mutex`
- `/debug/pprof/profile`
- `/debug/pprof/symbol`
- `/debug/pprof/threadcreate`
- `/debug/pprof/trace`
- `/extensions/available`
- `/extensions/enabled`
@ -153,8 +158,6 @@ Flags:
--case-sensitive use case-sensitive matching for filters
--code enable support for source code files
--code-theme string theme for source code syntax highlighting (default "solarized-dark256")
--compression string compression format to use for index (none, zlib, zstd) (default "zstd")
--compression-fast use fastest compression level (default is best)
--concurrency int maximum concurrency for scan threads (default 2147483647)
-d, --debug display even more verbose logs
--disable-buttons disable first/prev/next/last buttons

1
cmd/errors.go
View File

@ -16,7 +16,6 @@ import (
var (
ErrInvalidAdminPrefix = errors.New("admin path must match the pattern " + AllowedCharacters)
ErrInvalidCompression = errors.New("supported compression formats: none, zlib, zstd")
ErrInvalidConcurrency = errors.New("concurrency limit must be a positive integer")
ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit")
ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647")

4
cmd/files.go
View File

@ -454,11 +454,11 @@ func fileList(paths []string, filters *filters, sort string, index *fileIndex, f
case Index && !index.isEmpty() && !filters.isEmpty():
return filters.apply(index.List())
case Index && index.isEmpty() && !filters.isEmpty():
index.set(scanPaths(paths, sort, index, formats, errorChannel))
index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
return filters.apply(index.List())
case Index && index.isEmpty() && filters.isEmpty():
index.set(scanPaths(paths, sort, index, formats, errorChannel))
index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
return index.List()
case !Index && !filters.isEmpty():

90
cmd/index.go
View File

@ -5,7 +5,6 @@ Copyright © 2023 Seednode <seednode@seedno.de>
package cmd
import (
"compress/zlib"
"encoding/gob"
"fmt"
"io"
@ -63,7 +62,7 @@ func (index *fileIndex) remove(path string) {
index.mutex.Unlock()
}
func (index *fileIndex) set(val []string) {
func (index *fileIndex) set(val []string, errorChannel chan<- error) {
length := len(val)
if length < 1 {
@ -76,7 +75,7 @@ func (index *fileIndex) set(val []string) {
index.mutex.Unlock()
if Index && IndexFile != "" {
index.Export(IndexFile)
index.Export(IndexFile, errorChannel)
}
}
@ -94,53 +93,37 @@ func (index *fileIndex) isEmpty() bool {
return length == 0
}
func getReader(format string, file io.Reader) (io.ReadCloser, error) {
switch format {
case "none":
return io.NopCloser(file), nil
case "zlib":
return zlib.NewReader(file)
case "zstd":
func getReader(file io.Reader) (*zstd.Decoder, error) {
reader, err := zstd.NewReader(file)
if err != nil {
return io.NopCloser(file), err
return nil, err
}
return reader.IOReadCloser(), nil
return reader, nil
}
return io.NopCloser(file), ErrInvalidCompression
}
func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
switch {
case format == "none":
return file, nil
case format == "zlib" && CompressionFast:
return zlib.NewWriterLevel(file, zlib.BestSpeed)
case format == "zlib":
return zlib.NewWriterLevel(file, zlib.BestCompression)
case format == "zstd" && CompressionFast:
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest))
case format == "zstd":
func getWriter(file io.WriteCloser) (*zstd.Encoder, error) {
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
}
return file, ErrInvalidCompression
}
func (index *fileIndex) Export(path string) error {
func (index *fileIndex) Export(path string, errorChannel chan<- error) {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
errorChannel <- err
return
}
defer file.Close()
encoder, err := getWriter(Compression, file)
encoder, err := getWriter(file)
if err != nil {
return err
errorChannel <- err
encoder.Close()
return
}
defer encoder.Close()
@ -151,22 +134,22 @@ func (index *fileIndex) Export(path string) error {
if err != nil {
index.mutex.RUnlock()
return err
errorChannel <- err
return
}
length := len(index.list)
index.mutex.RUnlock()
// Close encoder prior to checking file size,
// to ensure the correct value is returned.
// If no compression is used, skip this step,
// as the encoder is just the file itself.
if Compression != "none" {
encoder.Close()
}
stats, err := file.Stat()
if err != nil {
return err
errorChannel <- err
return
}
if Verbose {
@ -178,27 +161,31 @@ func (index *fileIndex) Export(path string) error {
time.Since(startTime).Round(time.Microsecond),
)
}
return nil
}
func (index *fileIndex) Import(path string) error {
func (index *fileIndex) Import(path string, errorChannel chan<- error) {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
return err
errorChannel <- err
return
}
defer file.Close()
stats, err := file.Stat()
if err != nil {
return err
errorChannel <- err
return
}
reader, err := getReader(Compression, file)
reader, err := getReader(file)
if err != nil {
return err
errorChannel <- err
return
}
defer reader.Close()
@ -209,7 +196,9 @@ func (index *fileIndex) Import(path string) error {
if err != nil {
index.mutex.Unlock()
return err
errorChannel <- err
return
}
length := len(index.list)
index.mutex.Unlock()
@ -223,8 +212,6 @@ func (index *fileIndex) Import(path string) error {
time.Since(startTime).Round(time.Microsecond),
)
}
return nil
}
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
@ -256,10 +243,7 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
if IndexFile != "" {
err := index.Import(IndexFile)
if err == nil {
return
}
index.Import(IndexFile, errorChannel)
}
fileList(args, &filters{}, "", index, formats, errorChannel)

9
cmd/root.go
View File

@ -10,7 +10,6 @@ import (
"math"
"os"
"regexp"
"slices"
"time"
"github.com/spf13/cobra"
@ -18,7 +17,7 @@ import (
const (
AllowedCharacters string = `^[A-z0-9.\-_]+$`
ReleaseVersion string = "5.3.0"
ReleaseVersion string = "5.4.0"
)
var (
@ -31,8 +30,6 @@ var (
CaseSensitive bool
Code bool
CodeTheme string
Compression string
CompressionFast bool
Concurrency int
Debug bool
DisableButtons bool
@ -91,8 +88,6 @@ var (
return ErrInvalidConcurrency
case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
return ErrInvalidIgnoreFile
case !slices.Contains(CompressionFormats, Compression):
return ErrInvalidCompression
case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
return ErrInvalidAdminPrefix
case AdminPrefix != "":
@ -131,8 +126,6 @@ func init() {
rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (none, zlib, zstd)")
rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
rootCmd.Flags().IntVar(&Concurrency, "concurrency", 10240, "maximum concurrency for scan threads")
rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")