From 83809696ac3072ed32d4b64d8da4440fbcc39a95 Mon Sep 17 00:00:00 2001 From: Seednode Date: Tue, 30 Jan 2024 10:12:05 -0600 Subject: [PATCH] Change --info to --api; /index/rebuild now requires POST instead of GET request; merge --ignore and --ignore-file --- README.md | 33 ++++++++++++++++++-------------- cmd/files.go | 2 +- cmd/index.go | 27 -------------------------- cmd/info.go | 53 +++++++++++++++++++++++++++++++++++++++------------- cmd/root.go | 14 ++++++-------- cmd/web.go | 10 ++++------ 6 files changed, 70 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index d6e6d49..9dd5e07 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Dockerfile available [here](https://git.seedno.de/seednode/roulette/raw/branch/m An example instance with most features enabled can be found [here](https://nature.seedno.de/). ## Admin prefix -You can restrict access to certain functionality by prepending a secret string to the paths. +You can restrict access to certain functionality (the REST API and profiling endpoints) by prepending a secret string to the paths. For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`. @@ -44,6 +44,21 @@ The restricted paths are: While this might thwart very basic attacks, the proper solution for most use cases would likely be to add authentication via a reverse proxy. +## API +If the `--api` flag is passed, a number of REST endpoints are registered. + +The first of these—`/index/`—responds to GET requests with the contents of the index, in JSON format. + +The second—`/index/rebuild`—responds to POST requests by rebuilding the index. + +This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index. + +The remaining four endpoints respond to GET requests with information about the registered file types: +- `/extensions/available` +- `/extensions/enabled` +- `/types/available` +- `/types/enabled` + ## Filtering You can provide a comma-delimited string of alphanumeric patterns to match via the `include=` query parameter, assuming the `-f|--filter` flag is enabled. @@ -73,15 +88,6 @@ If `--index-file` is set, the index will be loaded from the specified file on st The index file consists of [zstd](https://facebook.github.io/zstd/)-compressed [gobs](https://pkg.go.dev/encoding/gob). -## Info -If the `-i|--info` flag is passed, five additional endpoints are registered. - -The first of these—`/index/`—returns the contents of the index, in JSON format. - -This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index. - -The remaining four endpoints—`/extensions/available`, `/extensions/enabled`, `/types/available` and `/types/enabled`—return information about the registered file types. - ## Refresh If the `--refresh` flag is passed and a positive-value `refresh=` query parameter is provided, the page will reload after that interval. @@ -143,6 +149,7 @@ Flags: --admin-prefix string string to prepend to administrative paths -a, --all enable all supported file types --allow-empty allow specifying paths containing no supported files + --api expose REST API --audio enable support for audio files --binary-prefix use IEC binary prefixes instead of SI decimal prefixes -b, --bind string address to bind to (default "0.0.0.0") @@ -158,13 +165,11 @@ Flags: --flash enable support for shockwave flash files (via ruffle.rs) --fun add a bit of excitement to your day -h, --help help for roulette - --ignore skip all directories containing a specified filename - --ignore-file string filename used to indicate directory should be skipped (default ".roulette-ignore") + --ignore string filename used to indicate directory should be skipped --images enable support for image files --index generate index of supported file paths at startup --index-file string path to optional persistent index file - --index-interval string interval at which to regenerate index (e.g. "3s" or "1h") - -i, --info expose informational endpoints + --index-interval string interval at which to regenerate index (e.g. "5m" or "1h") --max-file-count int skip directories with file counts above this value (default 2147483647) --min-file-count int skip directories with file counts below this value -p, --port int port to listen on (default 8080) diff --git a/cmd/files.go b/cmd/files.go index 42cc4b6..cdee63d 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -260,7 +260,7 @@ func walkPath(path string, fileChannel chan<- string, wg1 *sync.WaitGroup, stats if !node.IsDir() { files++ - if Ignore && node.Name() == IgnoreFile { + if Ignore != "" && node.Name() == Ignore { skipDir = true } } diff --git a/cmd/index.go b/cmd/index.go index b3c07d0..bb325f9 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -7,12 +7,10 @@ package cmd import ( "encoding/gob" "fmt" - "net/http" "os" "sync" "time" - "github.com/julienschmidt/httprouter" "github.com/klauspost/compress/zstd" "seedno.de/seednode/roulette/types" ) @@ -227,31 +225,6 @@ func rebuildIndex(args []string, index *fileIndex, formats types.Types, encoder fileList(args, &filters{}, "", index, formats, encoder, errorChannel) } -func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { - return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { - startTime := time.Now() - - rebuildIndex(args, index, formats, encoder, errorChannel) - - w.Header().Set("Content-Type", "text/plain;charset=UTF-8") - - _, err := w.Write([]byte("Ok\n")) - if err != nil { - errorChannel <- err - - return - } - - if Verbose { - fmt.Printf("%s | SERVE: Index rebuild requested by %s took %s\n", - startTime.Format(logDate), - realIP(r), - time.Since(startTime).Round(time.Microsecond), - ) - } - } -} - func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) { if IndexFile != "" { index.Import(IndexFile, errorChannel) diff --git a/cmd/info.go b/cmd/info.go index a1cdfbb..9d81066 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -13,9 +13,40 @@ import ( "time" "github.com/julienschmidt/httprouter" + "github.com/klauspost/compress/zstd" "seedno.de/seednode/roulette/types" ) +func serveExtensions(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + startTime := time.Now() + + w.Header().Set("Content-Type", "text/plain;charset=UTF-8") + + var extensions string + + if available { + extensions = types.SupportedFormats.GetExtensions() + } else { + extensions = formats.GetExtensions() + } + + written, err := w.Write([]byte(extensions)) + if err != nil { + errorChannel <- err + } + + if Verbose { + fmt.Printf("%s | SERVE: Registered extension list (%s) to %s in %s\n", + startTime.Format(logDate), + humanReadableSize(written), + realIP(r), + time.Since(startTime).Round(time.Microsecond), + ) + } + } +} + func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { startTime := time.Now() @@ -55,29 +86,24 @@ func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) http } } -func serveExtensions(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle { +func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { startTime := time.Now() + rebuildIndex(args, index, formats, encoder, errorChannel) + w.Header().Set("Content-Type", "text/plain;charset=UTF-8") - var extensions string - - if available { - extensions = types.SupportedFormats.GetExtensions() - } else { - extensions = formats.GetExtensions() - } - - written, err := w.Write([]byte(extensions)) + _, err := w.Write([]byte("Ok\n")) if err != nil { errorChannel <- err + + return } if Verbose { - fmt.Printf("%s | SERVE: Registered extension list (%s) to %s in %s\n", + fmt.Printf("%s | SERVE: Index rebuild requested by %s took %s\n", startTime.Format(logDate), - humanReadableSize(written), realIP(r), time.Since(startTime).Round(time.Microsecond), ) @@ -115,9 +141,10 @@ func serveMediaTypes(formats types.Types, available bool, errorChannel chan<- er } } -func registerInfoHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { +func registerAPIHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) { if Index { mux.GET(Prefix+AdminPrefix+"/index", serveIndex(args, index, errorChannel)) + mux.POST(Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel)) } mux.GET(Prefix+AdminPrefix+"/extensions/available", serveExtensions(formats, true, errorChannel)) diff --git a/cmd/root.go b/cmd/root.go index b50fb35..ddd84d1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,13 +17,14 @@ import ( const ( AllowedCharacters string = `^[A-z0-9.\-_]+$` - ReleaseVersion string = "6.4.3" + ReleaseVersion string = "7.0.0" ) var ( AdminPrefix string All bool AllowEmpty bool + API bool Audio bool BinaryPrefix bool Bind string @@ -38,13 +39,11 @@ var ( Filtering bool Flash bool Fun bool - Ignore bool - IgnoreFile string + Ignore string Images bool Index bool IndexFile string IndexInterval string - Info bool MaxFileCount int MinFileCount int Port int @@ -84,7 +83,7 @@ var ( return ErrInvalidPort case Concurrency < 1: return ErrInvalidConcurrency - case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile): + case Ignore != "" && !regexp.MustCompile(AllowedCharacters).MatchString(Ignore): return ErrInvalidIgnoreFile case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix): return ErrInvalidAdminPrefix @@ -118,6 +117,7 @@ func init() { rootCmd.Flags().StringVar(&AdminPrefix, "admin-prefix", "", "string to prepend to administrative paths") rootCmd.Flags().BoolVarP(&All, "all", "a", false, "enable all supported file types") rootCmd.Flags().BoolVar(&AllowEmpty, "allow-empty", false, "allow specifying paths containing no supported files") + rootCmd.Flags().BoolVar(&API, "api", false, "expose REST API") rootCmd.Flags().BoolVar(&Audio, "audio", false, "enable support for audio files") rootCmd.Flags().BoolVar(&BinaryPrefix, "binary-prefix", false, "use IEC binary prefixes instead of SI decimal prefixes") rootCmd.Flags().StringVarP(&Bind, "bind", "b", "0.0.0.0", "address to bind to") @@ -132,13 +132,11 @@ func init() { rootCmd.Flags().BoolVarP(&Filtering, "filter", "f", false, "enable filtering") rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)") rootCmd.Flags().BoolVar(&Fun, "fun", false, "add a bit of excitement to your day") - rootCmd.Flags().BoolVar(&Ignore, "ignore", false, "skip all directories containing a specified filename") - rootCmd.Flags().StringVar(&IgnoreFile, "ignore-file", ".roulette-ignore", "filename used to indicate directory should be skipped") + rootCmd.Flags().StringVar(&Ignore, "ignore", "", "filename used to indicate directory should be skipped") rootCmd.Flags().BoolVar(&Images, "images", false, "enable support for image files") rootCmd.Flags().BoolVar(&Index, "index", false, "generate index of supported file paths at startup") rootCmd.Flags().StringVar(&IndexFile, "index-file", "", "path to optional persistent index file") rootCmd.Flags().StringVar(&IndexInterval, "index-interval", "", "interval at which to regenerate index (e.g. \"5m\" or \"1h\")") - rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints") rootCmd.Flags().IntVar(&MaxFileCount, "max-file-count", math.MaxInt32, "skip directories with file counts above this value") rootCmd.Flags().IntVar(&MinFileCount, "min-file-count", 0, "skip directories with file counts below this value") rootCmd.Flags().IntVarP(&Port, "port", "p", 8080, "port to listen on") diff --git a/cmd/web.go b/cmd/web.go index 53f5f3a..e50bfcf 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -584,9 +584,11 @@ func ServePage(args []string) error { quit := make(chan struct{}) defer close(quit) - if Index { - mux.GET(Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel)) + if API { + registerAPIHandlers(mux, args, index, formats, encoder, errorChannel) + } + if Index { importIndex(paths, index, formats, encoder, errorChannel) if IndexInterval != "" { @@ -594,10 +596,6 @@ func ServePage(args []string) error { } } - if Info { - registerInfoHandlers(mux, args, index, formats, errorChannel) - } - if Profile { registerProfileHandlers(mux) }