Compare commits
No commits in common. "7546c6257ffa4ef2df624ba98e0815b1b1a72dce" and "4e85621335996136a751d7f291d1242475aabf12" have entirely different histories.
7546c6257f
...
4e85621335
|
|
@ -25,15 +25,10 @@ You can restrict access to certain functionality by prepending a secret string t
|
|||
For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`.
|
||||
|
||||
The restricted paths are:
|
||||
- `/debug/pprof/allocs`
|
||||
- `/debug/pprof/block`
|
||||
- `/debug/pprof/`
|
||||
- `/debug/pprof/cmdline`
|
||||
- `/debug/pprof/goroutine`
|
||||
- `/debug/pprof/heap`
|
||||
- `/debug/pprof/mutex`
|
||||
- `/debug/pprof/profile`
|
||||
- `/debug/pprof/symbol`
|
||||
- `/debug/pprof/threadcreate`
|
||||
- `/debug/pprof/trace`
|
||||
- `/extensions/available`
|
||||
- `/extensions/enabled`
|
||||
|
|
@ -158,6 +153,8 @@ Flags:
|
|||
--case-sensitive use case-sensitive matching for filters
|
||||
--code enable support for source code files
|
||||
--code-theme string theme for source code syntax highlighting (default "solarized-dark256")
|
||||
--compression string compression format to use for index (none, zlib, zstd) (default "zstd")
|
||||
--compression-fast use fastest compression level (default is best)
|
||||
--concurrency int maximum concurrency for scan threads (default 2147483647)
|
||||
-d, --debug display even more verbose logs
|
||||
--disable-buttons disable first/prev/next/last buttons
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import (
|
|||
|
||||
var (
|
||||
ErrInvalidAdminPrefix = errors.New("admin path must match the pattern " + AllowedCharacters)
|
||||
ErrInvalidCompression = errors.New("supported compression formats: none, zlib, zstd")
|
||||
ErrInvalidConcurrency = errors.New("concurrency limit must be a positive integer")
|
||||
ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit")
|
||||
ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647")
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"seedno.de/seednode/roulette/types"
|
||||
)
|
||||
|
||||
|
|
@ -448,18 +447,18 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
|
|||
return list
|
||||
}
|
||||
|
||||
func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) []string {
|
||||
func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
|
||||
switch {
|
||||
case Index && !index.isEmpty() && filters.isEmpty():
|
||||
return index.List()
|
||||
case Index && !index.isEmpty() && !filters.isEmpty():
|
||||
return filters.apply(index.List())
|
||||
case Index && index.isEmpty() && !filters.isEmpty():
|
||||
index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
|
||||
index.set(scanPaths(paths, sort, index, formats, errorChannel))
|
||||
|
||||
return filters.apply(index.List())
|
||||
case Index && index.isEmpty() && filters.isEmpty():
|
||||
index.set(scanPaths(paths, sort, index, formats, errorChannel), encoder, errorChannel)
|
||||
index.set(scanPaths(paths, sort, index, formats, errorChannel))
|
||||
|
||||
return index.List()
|
||||
case !Index && !filters.isEmpty():
|
||||
|
|
|
|||
111
cmd/index.go
111
cmd/index.go
|
|
@ -5,11 +5,12 @@ Copyright © 2023 Seednode <seednode@seedno.de>
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"compress/zlib"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -18,6 +19,12 @@ import (
|
|||
"seedno.de/seednode/roulette/types"
|
||||
)
|
||||
|
||||
var CompressionFormats = []string{
|
||||
"none",
|
||||
"zlib",
|
||||
"zstd",
|
||||
}
|
||||
|
||||
type fileIndex struct {
|
||||
mutex *sync.RWMutex
|
||||
list []string
|
||||
|
|
@ -56,7 +63,7 @@ func (index *fileIndex) remove(path string) {
|
|||
index.mutex.Unlock()
|
||||
}
|
||||
|
||||
func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel chan<- error) {
|
||||
func (index *fileIndex) set(val []string) {
|
||||
length := len(val)
|
||||
|
||||
if length < 1 {
|
||||
|
|
@ -69,7 +76,7 @@ func (index *fileIndex) set(val []string, encoder *zstd.Encoder, errorChannel ch
|
|||
index.mutex.Unlock()
|
||||
|
||||
if Index && IndexFile != "" {
|
||||
index.Export(IndexFile, encoder, errorChannel)
|
||||
index.Export(IndexFile)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -87,19 +94,54 @@ func (index *fileIndex) isEmpty() bool {
|
|||
return length == 0
|
||||
}
|
||||
|
||||
func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel chan<- error) {
|
||||
func getReader(format string, file io.Reader) (io.ReadCloser, error) {
|
||||
switch format {
|
||||
case "none":
|
||||
return io.NopCloser(file), nil
|
||||
case "zlib":
|
||||
return zlib.NewReader(file)
|
||||
case "zstd":
|
||||
reader, err := zstd.NewReader(file)
|
||||
if err != nil {
|
||||
return io.NopCloser(file), err
|
||||
}
|
||||
|
||||
return reader.IOReadCloser(), nil
|
||||
}
|
||||
|
||||
return io.NopCloser(file), ErrInvalidCompression
|
||||
}
|
||||
|
||||
func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
|
||||
switch {
|
||||
case format == "none":
|
||||
return file, nil
|
||||
case format == "zlib" && CompressionFast:
|
||||
return zlib.NewWriterLevel(file, zlib.BestSpeed)
|
||||
case format == "zlib":
|
||||
return zlib.NewWriterLevel(file, zlib.BestCompression)
|
||||
case format == "zstd" && CompressionFast:
|
||||
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest))
|
||||
case format == "zstd":
|
||||
return zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
||||
}
|
||||
|
||||
return file, ErrInvalidCompression
|
||||
}
|
||||
|
||||
func (index *fileIndex) Export(path string) error {
|
||||
startTime := time.Now()
|
||||
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
errorChannel <- err
|
||||
|
||||
return
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
encoder.Reset(file)
|
||||
|
||||
encoder, err := getWriter(Compression, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer encoder.Close()
|
||||
|
||||
enc := gob.NewEncoder(encoder)
|
||||
|
|
@ -109,22 +151,22 @@ func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel
|
|||
if err != nil {
|
||||
index.mutex.RUnlock()
|
||||
|
||||
errorChannel <- err
|
||||
|
||||
return
|
||||
return err
|
||||
}
|
||||
length := len(index.list)
|
||||
index.mutex.RUnlock()
|
||||
|
||||
// Close encoder prior to checking file size,
|
||||
// to ensure the correct value is returned.
|
||||
// If no compression is used, skip this step,
|
||||
// as the encoder is just the file itself.
|
||||
if Compression != "none" {
|
||||
encoder.Close()
|
||||
}
|
||||
|
||||
stats, err := file.Stat()
|
||||
if err != nil {
|
||||
errorChannel <- err
|
||||
|
||||
return
|
||||
return err
|
||||
}
|
||||
|
||||
if Verbose {
|
||||
|
|
@ -136,31 +178,27 @@ func (index *fileIndex) Export(path string, encoder *zstd.Encoder, errorChannel
|
|||
time.Since(startTime).Round(time.Microsecond),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (index *fileIndex) Import(path string, errorChannel chan<- error) {
|
||||
func (index *fileIndex) Import(path string) error {
|
||||
startTime := time.Now()
|
||||
|
||||
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
|
||||
if err != nil {
|
||||
errorChannel <- err
|
||||
|
||||
return
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stats, err := file.Stat()
|
||||
if err != nil {
|
||||
errorChannel <- err
|
||||
|
||||
return
|
||||
return err
|
||||
}
|
||||
|
||||
reader, err := zstd.NewReader(file)
|
||||
reader, err := getReader(Compression, file)
|
||||
if err != nil {
|
||||
errorChannel <- err
|
||||
|
||||
return
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
|
|
@ -171,9 +209,7 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
|
|||
if err != nil {
|
||||
index.mutex.Unlock()
|
||||
|
||||
errorChannel <- err
|
||||
|
||||
return
|
||||
return err
|
||||
}
|
||||
length := len(index.list)
|
||||
index.mutex.Unlock()
|
||||
|
|
@ -187,15 +223,17 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
|
|||
time.Since(startTime).Round(time.Microsecond),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
|
||||
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
|
||||
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||
startTime := time.Now()
|
||||
|
||||
index.clear()
|
||||
|
||||
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
|
||||
fileList(args, &filters{}, "", index, formats, errorChannel)
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
|
||||
|
|
@ -213,15 +251,16 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, enc
|
|||
time.Since(startTime).Round(time.Microsecond),
|
||||
)
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
}
|
||||
}
|
||||
|
||||
func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
|
||||
func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
|
||||
if IndexFile != "" {
|
||||
index.Import(IndexFile, errorChannel)
|
||||
err := index.Import(IndexFile)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
|
||||
fileList(args, &filters{}, "", index, formats, errorChannel)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,20 +5,34 @@ Copyright © 2023 Seednode <seednode@seedno.de>
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
|
||||
func registerProfileHandlers(mux *httprouter.Router) {
|
||||
mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/allocs", pprof.Handler("allocs"))
|
||||
mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/block", pprof.Handler("block"))
|
||||
mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/goroutine", pprof.Handler("goroutine"))
|
||||
mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/heap", pprof.Handler("heap"))
|
||||
mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/mutex", pprof.Handler("mutex"))
|
||||
mux.Handler("GET", Prefix+AdminPrefix+"/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
|
||||
mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/cmdline", pprof.Cmdline)
|
||||
mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/profile", pprof.Profile)
|
||||
mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/symbol", pprof.Symbol)
|
||||
mux.HandlerFunc("GET", Prefix+AdminPrefix+"/debug/pprof/trace", pprof.Trace)
|
||||
func registerProfileHandler(mux *httprouter.Router, verb, path string, handler http.HandlerFunc) {
|
||||
mux.HandlerFunc(verb, path, handler)
|
||||
|
||||
if Redact && AdminPrefix != "" {
|
||||
path = strings.ReplaceAll(path, AdminPrefix, "/<admin_prefix>")
|
||||
}
|
||||
|
||||
if Handlers {
|
||||
fmt.Printf("%s | SERVE: Registered handler for %s\n",
|
||||
time.Now().Format(logDate),
|
||||
path,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func registerProfileHandlers(mux *httprouter.Router) {
|
||||
registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/", pprof.Index)
|
||||
registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/cmdline", pprof.Cmdline)
|
||||
registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/profile", pprof.Profile)
|
||||
registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/symbol", pprof.Symbol)
|
||||
registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/trace", pprof.Trace)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
|
@ -17,7 +18,7 @@ import (
|
|||
|
||||
const (
|
||||
AllowedCharacters string = `^[A-z0-9.\-_]+$`
|
||||
ReleaseVersion string = "5.4.3"
|
||||
ReleaseVersion string = "5.2.0"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -30,6 +31,8 @@ var (
|
|||
CaseSensitive bool
|
||||
Code bool
|
||||
CodeTheme string
|
||||
Compression string
|
||||
CompressionFast bool
|
||||
Concurrency int
|
||||
Debug bool
|
||||
DisableButtons bool
|
||||
|
|
@ -88,6 +91,8 @@ var (
|
|||
return ErrInvalidConcurrency
|
||||
case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
|
||||
return ErrInvalidIgnoreFile
|
||||
case !slices.Contains(CompressionFormats, Compression):
|
||||
return ErrInvalidCompression
|
||||
case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
|
||||
return ErrInvalidAdminPrefix
|
||||
case AdminPrefix != "":
|
||||
|
|
@ -126,6 +131,8 @@ func init() {
|
|||
rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
|
||||
rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
|
||||
rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
|
||||
rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (none, zlib, zstd)")
|
||||
rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
|
||||
rootCmd.Flags().IntVar(&Concurrency, "concurrency", 10240, "maximum concurrency for scan threads")
|
||||
rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
|
||||
rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")
|
||||
|
|
|
|||
18
cmd/web.go
18
cmd/web.go
|
|
@ -21,7 +21,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/yosssi/gohtml"
|
||||
"seedno.de/seednode/roulette/types"
|
||||
"seedno.de/seednode/roulette/types/audio"
|
||||
|
|
@ -32,8 +31,6 @@ import (
|
|||
"seedno.de/seednode/roulette/types/video"
|
||||
)
|
||||
|
||||
var ()
|
||||
|
||||
const (
|
||||
logDate string = `2006-01-02T15:04:05.000-07:00`
|
||||
sourcePrefix string = `/source`
|
||||
|
|
@ -171,7 +168,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error
|
|||
}
|
||||
}
|
||||
|
||||
func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
|
||||
func serveRoot(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
|
||||
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||
refererUri, err := stripQueryParams(refererToUri(r.Referer()))
|
||||
if err != nil {
|
||||
|
|
@ -206,7 +203,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z
|
|||
}
|
||||
}
|
||||
|
||||
list := fileList(paths, filters, sortOrder, index, formats, encoder, errorChannel)
|
||||
list := fileList(paths, filters, sortOrder, index, formats, errorChannel)
|
||||
|
||||
loop:
|
||||
for timeout := time.After(timeout); ; {
|
||||
|
|
@ -579,12 +576,7 @@ func ServePage(args []string) error {
|
|||
}
|
||||
}()
|
||||
|
||||
encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel))
|
||||
registerHandler(mux, Prefix, serveRoot(paths, index, formats, errorChannel))
|
||||
|
||||
Prefix = strings.TrimSuffix(Prefix, "/")
|
||||
|
||||
|
|
@ -603,9 +595,9 @@ func ServePage(args []string) error {
|
|||
registerHandler(mux, Prefix+"/version", serveVersion(errorChannel))
|
||||
|
||||
if Index {
|
||||
registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel))
|
||||
registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel))
|
||||
|
||||
importIndex(paths, index, formats, encoder, errorChannel)
|
||||
importIndex(paths, index, formats, errorChannel)
|
||||
}
|
||||
|
||||
if Info {
|
||||
|
|
|
|||
Loading…
Reference in New Issue