From 14adbdf742c5d28b47f8eadfc1d050ed9dd0910d Mon Sep 17 00:00:00 2001 From: Seednode Date: Sun, 17 Dec 2023 06:35:03 -0600 Subject: [PATCH] Add concurrency limits; add optional admin path prefix --- cmd/errors.go | 2 ++ cmd/files.go | 15 ++++++++++++--- cmd/info.go | 16 ++++++++-------- cmd/profile.go | 10 +++++----- cmd/root.go | 13 ++++++++++++- cmd/web.go | 2 +- 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/cmd/errors.go b/cmd/errors.go index 718dbbe..c6502b2 100644 --- a/cmd/errors.go +++ b/cmd/errors.go @@ -16,6 +16,8 @@ import ( ) var ( + ErrInvalidAdminPrefix = errors.New("admin path must not contain a '/'") + ErrInvalidConcurrency = errors.New("concurrency limit must be between 1 and 8192 inclusive") ErrInvalidFileCountRange = errors.New("maximum file count limit must be greater than or equal to minimum file count limit") ErrInvalidFileCountValue = errors.New("file count limits must be non-negative integers no greater than 2147483647") ErrInvalidPort = errors.New("listen port must be an integer between 1 and 65535 inclusive") diff --git a/cmd/files.go b/cmd/files.go index f8b949d..eed8c14 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -226,7 +226,13 @@ func hasSupportedFiles(path string, formats types.Types) (bool, error) { } } -func walkPath(path string, fileChannel chan<- string, stats *scanStatsChannels, formats types.Types) error { +func walkPath(path string, fileChannel chan<- string, stats *scanStatsChannels, limit chan int, formats types.Types) error { + limit <- 1 + + defer func() { + <-limit + }() + errorChannel := make(chan error) done := make(chan bool, 1) @@ -277,7 +283,7 @@ func walkPath(path string, fileChannel chan<- string, stats *scanStatsChannels, switch { case node.IsDir() && Recursive: - err := walkPath(fullPath, fileChannel, stats, formats) + err := walkPath(fullPath, fileChannel, stats, limit, formats) if err != nil { errorChannel <- err @@ -398,6 +404,8 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type } }() + limit := make(chan int, Concurrency) + var wg sync.WaitGroup for i := 0; i < len(paths); i++ { @@ -407,7 +415,8 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type defer func() { wg.Done() }() - err := walkPath(paths[i], fileChannel, statsChannels, formats) + + err := walkPath(paths[i], fileChannel, statsChannels, limit, formats) if err != nil { errorChannel <- err diff --git a/cmd/info.go b/cmd/info.go index e13062b..2e6e5cf 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -305,19 +305,19 @@ func serveEnabledMediaTypes(formats types.Types, errorChannel chan<- error) http func registerInfoHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { if Index { - registerHandler(mux, Prefix+"/index/html", serveIndexHtml(args, index, false)) + registerHandler(mux, Prefix+AdminPrefix+"/index/html", serveIndexHtml(args, index, false)) if PageLength != 0 { - registerHandler(mux, Prefix+"/index/html/:page", serveIndexHtml(args, index, true)) + registerHandler(mux, Prefix+AdminPrefix+"/index/html/:page", serveIndexHtml(args, index, true)) } - registerHandler(mux, Prefix+"/index/json", serveIndexJson(args, index, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/index/json", serveIndexJson(args, index, errorChannel)) if PageLength != 0 { - registerHandler(mux, Prefix+"/index/json/:page", serveIndexJson(args, index, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/index/json/:page", serveIndexJson(args, index, errorChannel)) } } - registerHandler(mux, Prefix+"/extensions/available", serveAvailableExtensions(errorChannel)) - registerHandler(mux, Prefix+"/extensions/enabled", serveEnabledExtensions(formats, errorChannel)) - registerHandler(mux, Prefix+"/types/available", serveAvailableMediaTypes(errorChannel)) - registerHandler(mux, Prefix+"/types/enabled", serveEnabledMediaTypes(formats, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/extensions/available", serveAvailableExtensions(errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/extensions/enabled", serveEnabledExtensions(formats, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/types/available", serveAvailableMediaTypes(errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/types/enabled", serveEnabledMediaTypes(formats, errorChannel)) } diff --git a/cmd/profile.go b/cmd/profile.go index cc0c8da..90bec5c 100644 --- a/cmd/profile.go +++ b/cmd/profile.go @@ -25,9 +25,9 @@ func registerProfileHandler(mux *httprouter.Router, verb, path string, handler h } func registerProfileHandlers(mux *httprouter.Router) { - registerProfileHandler(mux, "GET", Prefix+"/debug/pprof/", pprof.Index) - registerProfileHandler(mux, "GET", Prefix+"/debug/pprof/cmdline", pprof.Cmdline) - registerProfileHandler(mux, "GET", Prefix+"/debug/pprof/profile", pprof.Profile) - registerProfileHandler(mux, "GET", Prefix+"/debug/pprof/symbol", pprof.Symbol) - registerProfileHandler(mux, "GET", Prefix+"/debug/pprof/trace", pprof.Trace) + registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/", pprof.Index) + registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/cmdline", pprof.Cmdline) + registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/profile", pprof.Profile) + registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/symbol", pprof.Symbol) + registerProfileHandler(mux, "GET", Prefix+AdminPrefix+"/debug/pprof/trace", pprof.Trace) } diff --git a/cmd/root.go b/cmd/root.go index 7d0c696..4b1a10e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -7,21 +7,24 @@ package cmd import ( "log" "math" + "strings" "github.com/spf13/cobra" ) const ( - ReleaseVersion string = "3.3.2" + ReleaseVersion string = "3.4.0" ) var ( + AdminPrefix string All bool Audio bool Bind string CaseSensitive bool Code bool CodeTheme string + Concurrency int DisableButtons bool ExitOnError bool Fallback bool @@ -61,6 +64,12 @@ var ( return ErrInvalidFileCountRange case Port < 1 || Port > 65535: return ErrInvalidPort + case Concurrency < 1 || Concurrency > 8192: + return ErrInvalidConcurrency + case strings.Contains(AdminPrefix, "/"): + return ErrInvalidAdminPrefix + case AdminPrefix != "": + AdminPrefix = "/" + AdminPrefix } return nil @@ -84,12 +93,14 @@ func Execute() { } func init() { + rootCmd.Flags().StringVar(&AdminPrefix, "admin-prefix", "", "string to prepend to administrative paths") rootCmd.Flags().BoolVarP(&All, "all", "a", false, "enable all supported file types") rootCmd.Flags().BoolVar(&Audio, "audio", false, "enable support for audio files") rootCmd.Flags().StringVarP(&Bind, "bind", "b", "0.0.0.0", "address to bind to") rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters") rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files") rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting") + rootCmd.Flags().IntVar(&Concurrency, "concurrency", 1024, "maximum concurrency for scan threads") rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons") rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing the error") rootCmd.Flags().BoolVar(&Fallback, "fallback", false, "serve files as application/octet-stream if no matching format is registered") diff --git a/cmd/web.go b/cmd/web.go index c318233..a8b9e28 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -544,7 +544,7 @@ func ServePage(args []string) error { registerHandler(mux, Prefix+"/version", serveVersion()) if Index { - registerHandler(mux, Prefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel)) err = importIndex(paths, index, formats) if err != nil {