SeanOMik
/
roulette
mirror of https://git.seedno.de/seednode/roulette.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove non-zstd compression options

This commit is contained in:
Seednode 2024-01-14 09:29:22 -06:00
parent 7868f0d7ad
commit 2f06ae3605
5 changed files with 89 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -25,10 +25,15 @@ You can restrict access to certain functionality by prepending a secret string t
For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`.
The restricted paths are:
- `/debug/pprof/`
- `/debug/pprof/allocs`
- `/debug/pprof/block`
- `/debug/pprof/cmdline`
- `/debug/pprof/goroutine`
- `/debug/pprof/heap`
- `/debug/pprof/mutex`
- `/debug/pprof/profile`
- `/debug/pprof/symbol`
- `/debug/pprof/threadcreate`
- `/debug/pprof/trace`
- `/extensions/available`
- `/extensions/enabled`
@ -153,8 +158,6 @@ Flags:
--case-sensitive use case-sensitive matching for filters
--code enable support for source code files
--code-theme string theme for source code syntax highlighting (default "solarized-dark256")
--compression string compression format to use for index (none, zlib, zstd) (default "zstd")
--compression-fast use fastest compression level (default is best)
--concurrency int maximum concurrency for scan threads (default 2147483647)
-d, --debug display even more verbose logs
--disable-buttons disable first/prev/next/last buttons

1
cmd/errors.go
View File

@ -16,7 +16,6 @@ import (
var (
ErrInvalidAdminPrefix = errors.New("admin path must match the pattern " + AllowedCharacters)
ErrInvalidCompression = errors.New("supported compression formats: none, zlib, zstd")
ErrInvalidConcurrency = errors.New("concurrency limit must be a positive integer")
ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit")
ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647")

4
cmd/files.go
View File

@ -454,11 +454,11 @@ func fileList(paths []string, filters *filters, sort string, index *fileIndex, f
case Index && !index.isEmpty() && !filters.isEmpty():
return filters.apply(index.List())
case Index && index.isEmpty() && !filters.isEmpty():
index.set(scanPaths(paths, sort, index, formats, errorChannel))
index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
return filters.apply(index.List())
case Index && index.isEmpty() && filters.isEmpty():
index.set(scanPaths(paths, sort, index, formats, errorChannel))
index.set(scanPaths(paths, sort, index, formats, errorChannel), errorChannel)
return index.List()
case !Index && !filters.isEmpty():

98
cmd/index.go
View File

@ -5,7 +5,6 @@ Copyright © 2023 Seednode <seednode@seedno.de>
package cmd
import (
"compress/zlib"
"encoding/gob"
"fmt"
"io"
@ -63,7 +62,7 @@ func (index *fileIndex) remove(path string) {
index.mutex.Unlock()
}
func (index *fileIndex) set(val []string) {
func (index *fileIndex) set(val []string, errorChannel chan<- error) {
length := len(val)
if length < 1 {
@ -76,7 +75,7 @@ func (index *fileIndex) set(val []string) {
index.mutex.Unlock()
if Index && IndexFile != "" {
index.Export(IndexFile)
index.Export(IndexFile, errorChannel)
}
}
@ -94,53 +93,37 @@ func (index *fileIndex) isEmpty() bool {
return length == 0
}
func getReader(format string, file io.Reader) (io.ReadCloser, error) {
switch format {
case "none":
return io.NopCloser(file), nil
case "zlib":
return zlib.NewReader(file)
case "zstd":
reader, err := zstd.NewReader(file)
if err != nil {
return io.NopCloser(file), err
}
return reader.IOReadCloser(), nil
func getReader(file io.Reader) (*zstd.Decoder, error) {
reader, err := zstd.NewReader(file)
if err != nil {
return nil, err
}
return io.NopCloser(file), ErrInvalidCompression
return reader, nil
}
func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
switch {
case format == "none":
return file, nil
case format == "zlib" && CompressionFast:
return zlib.NewWriterLevel(file, zlib.BestSpeed)
case format == "zlib":
return zlib.NewWriterLevel(file, zlib.BestCompression)
case format == "zstd" && CompressionFast:
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest))
case format == "zstd":
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
}
return file, ErrInvalidCompression
func getWriter(file io.WriteCloser) (*zstd.Encoder, error) {
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
}
func (index *fileIndex) Export(path string) error {
func (index *fileIndex) Export(path string, errorChannel chan<- error) {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
errorChannel <- err
return
}
defer file.Close()
encoder, err := getWriter(Compression, file)
encoder, err := getWriter(file)
if err != nil {
return err
errorChannel <- err
encoder.Close()
return
}
defer encoder.Close()
@ -151,22 +134,22 @@ func (index *fileIndex) Export(path string) error {
if err != nil {
index.mutex.RUnlock()
return err
errorChannel <- err
return
}
length := len(index.list)
index.mutex.RUnlock()
// Close encoder prior to checking file size,
// to ensure the correct value is returned.
// If no compression is used, skip this step,
// as the encoder is just the file itself.
if Compression != "none" {
encoder.Close()
}
encoder.Close()
stats, err := file.Stat()
if err != nil {
return err
errorChannel <- err
return
}
if Verbose {
@ -178,27 +161,31 @@ func (index *fileIndex) Export(path string) error {
time.Since(startTime).Round(time.Microsecond),
)
}
return nil
}
func (index *fileIndex) Import(path string) error {
func (index *fileIndex) Import(path string, errorChannel chan<- error) {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
return err
errorChannel <- err
return
}
defer file.Close()
stats, err := file.Stat()
if err != nil {
return err
errorChannel <- err
return
}
reader, err := getReader(Compression, file)
reader, err := getReader(file)
if err != nil {
return err
errorChannel <- err
return
}
defer reader.Close()
@ -209,7 +196,9 @@ func (index *fileIndex) Import(path string) error {
if err != nil {
index.mutex.Unlock()
return err
errorChannel <- err
return
}
length := len(index.list)
index.mutex.Unlock()
@ -223,8 +212,6 @@ func (index *fileIndex) Import(path string) error {
time.Since(startTime).Round(time.Microsecond),
)
}
return nil
}
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
@ -256,10 +243,7 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
if IndexFile != "" {
err := index.Import(IndexFile)
if err == nil {
return
}
index.Import(IndexFile, errorChannel)
}
fileList(args, &filters{}, "", index, formats, errorChannel)

87
cmd/root.go
View File

@ -10,7 +10,6 @@ import (
"math"
"os"
"regexp"
"slices"
"time"
"github.com/spf13/cobra"
@ -18,51 +17,49 @@ import (
const (
AllowedCharacters string = `^[A-z0-9.\-_]+$`
ReleaseVersion string = "5.3.0"
ReleaseVersion string = "5.4.0"
)
var (
AdminPrefix string
All bool
AllowEmpty bool
Audio bool
BinaryPrefix bool
Bind string
CaseSensitive bool
Code bool
CodeTheme string
Compression string
CompressionFast bool
Concurrency int
Debug bool
DisableButtons bool
ExitOnError bool
Fallback bool
Filtering bool
Flash bool
Fun bool
Handlers bool
Ignore bool
IgnoreFile string
Images bool
Index bool
IndexFile string
Info bool
MaxFileCount int
MinFileCount int
PageLength int
Port int
Prefix string
Profile bool
Recursive bool
Redact bool
Refresh bool
Russian bool
Sorting bool
Text bool
Verbose bool
Version bool
Videos bool
AdminPrefix string
All bool
AllowEmpty bool
Audio bool
BinaryPrefix bool
Bind string
CaseSensitive bool
Code bool
CodeTheme string
Concurrency int
Debug bool
DisableButtons bool
ExitOnError bool
Fallback bool
Filtering bool
Flash bool
Fun bool
Handlers bool
Ignore bool
IgnoreFile string
Images bool
Index bool
IndexFile string
Info bool
MaxFileCount int
MinFileCount int
PageLength int
Port int
Prefix string
Profile bool
Recursive bool
Redact bool
Refresh bool
Russian bool
Sorting bool
Text bool
Verbose bool
Version bool
Videos bool
RequiredArgs = []string{
"all",
@ -91,8 +88,6 @@ var (
return ErrInvalidConcurrency
case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
return ErrInvalidIgnoreFile
case !slices.Contains(CompressionFormats, Compression):
return ErrInvalidCompression
case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
return ErrInvalidAdminPrefix
case AdminPrefix != "":
@ -131,8 +126,6 @@ func init() {
rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (none, zlib, zstd)")
rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
rootCmd.Flags().IntVar(&Concurrency, "concurrency", 10240, "maximum concurrency for scan threads")
rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")