Change --info to --api; /index/rebuild now requires POST instead of GET request; merge --ignore and --ignore-file

README.md

@@ -20,7 +20,7 @@ Dockerfile available [here](https://git.seedno.de/seednode/roulette/raw/branch/m
 An example instance with most features enabled can be found [here](https://nature.seedno.de/).
 
 ## Admin prefix
-You can restrict access to certain functionality by prepending a secret string to the paths.
+You can restrict access to certain functionality (the REST API and profiling endpoints) by prepending a secret string to the paths.
 
 For example, providing the `--admin-prefix=abc123` flag will register the index rebuild path as `/abc123/index/rebuild`.
 
@@ -44,6 +44,21 @@ The restricted paths are:
 
 While this might thwart very basic attacks, the proper solution for most use cases would likely be to add authentication via a reverse proxy.
 
+## API
+If the `--api` flag is passed, a number of REST endpoints are registered.
+
+The first of these—`/index/`—responds to GET requests with the contents of the index, in JSON format.
+
+The second—`/index/rebuild`—responds to POST requests by rebuilding the index.
+
+This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index.
+
+The remaining four endpoints respond to GET requests with information about the registered file types:
+- `/extensions/available`
+- `/extensions/enabled`
+- `/types/available`
+- `/types/enabled`
+
 ## Filtering
 You can provide a comma-delimited string of alphanumeric patterns to match via the `include=` query parameter, assuming the `-f|--filter` flag is enabled.
 
@@ -67,24 +82,11 @@ This will slightly increase the delay before the application begins responding t
 
 The index can be regenerated at any time by accessing the `/index/rebuild` endpoint.
 
+Automatic index rebuilds can be enabled via the `--index-interval` flag, which accepts [time.Duration](https://pkg.go.dev/time#ParseDuration) strings.
+
 If `--index-file` is set, the index will be loaded from the specified file on start, and written to the file whenever it is re-generated.
 
-The index file consists of (optionally compressed) [gobs](https://pkg.go.dev/encoding/gob).
+The index file consists of [zstd](https://facebook.github.io/zstd/)-compressed [gobs](https://pkg.go.dev/encoding/gob).
-
-The compression format can be specified via the `--compression` flag.
-
-Supported formats are `none`, `zlib`, and `zstd`.
-
-Optionally, `--compression-fast` can be used to use the fastest instead of the best compression mode.
-
-## Info
-If the `-i|--info` flag is passed, five additional endpoints are registered.
-
-The first of these—`/index/`—returns the contents of the index, in JSON format.
-
-This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index.
-
-The remaining four endpoints—`/extensions/available`, `/extensions/enabled`, `/types/available` and `/types/enabled`—return information about the registered file types.
 
 ## Refresh
 If the `--refresh` flag is passed and a positive-value `refresh=<integer><unit>` query parameter is provided, the page will reload after that interval.
@@ -102,7 +104,7 @@ If the `--russian` flag is passed, everything functions exactly as you would exp
 
 That is, files will be deleted after being served. This is not a joke, you *will* lose data.
 
-This uses `os.Remove()` and checks to ensure the specified file is inside one of the paths passed to `roulette`.
+This uses [os.Remove()](https://pkg.go.dev/os#Remove) and checks to ensure the specified file is inside one of the paths passed to `roulette`.
 
 That said, this has not been tested to any real extent, so only pass this flag on systems you don't care about.
 
@@ -147,6 +149,7 @@ Flags:
       --admin-prefix string     string to prepend to administrative paths
   -a, --all                     enable all supported file types
       --allow-empty             allow specifying paths containing no supported files
+      --api                     expose REST API
       --audio                   enable support for audio files
       --binary-prefix           use IEC binary prefixes instead of SI decimal prefixes
   -b, --bind string             address to bind to (default "0.0.0.0")
@@ -162,12 +165,11 @@ Flags:
       --flash                   enable support for shockwave flash files (via ruffle.rs)
       --fun                     add a bit of excitement to your day
   -h, --help                    help for roulette
-      --ignore                skip all directories containing a specified filename
+      --ignore string           filename used to indicate directory should be skipped
-      --ignore-file string    filename used to indicate directory should be skipped (default ".roulette-ignore")
       --images                  enable support for image files
       --index                   generate index of supported file paths at startup
       --index-file string       path to optional persistent index file
-  -i, --info                  expose informational endpoints
+      --index-interval string   interval at which to regenerate index (e.g. "5m" or "1h")
       --max-file-count int      skip directories with file counts above this value (default 2147483647)
       --min-file-count int      skip directories with file counts below this value
   -p, --port int                port to listen on (default 8080)

cmd/files.go

@@ -260,7 +260,7 @@ func walkPath(path string, fileChannel chan<- string, wg1 *sync.WaitGroup, stats
 		if !node.IsDir() {
 			files++
 
-			if Ignore && node.Name() == IgnoreFile {
+			if Ignore != "" && node.Name() == Ignore {
 				skipDir = true
 			}
 		}

cmd/index.go

@@ -7,12 +7,10 @@ package cmd
 import (
 	"encoding/gob"
 	"fmt"
-	"net/http"
 	"os"
 	"sync"
 	"time"
 
-	"github.com/julienschmidt/httprouter"
 	"github.com/klauspost/compress/zstd"
 	"seedno.de/seednode/roulette/types"
 )
@@ -188,32 +186,44 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
 	}
 }
 
-func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
+func registerIndexInterval(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, quit <-chan struct{}, errorChannel chan<- error) {
-	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+	interval, err := time.ParseDuration(IndexInterval)
-		startTime := time.Now()
-
-		index.clear()
-
-		fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
-
-		w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
-
-		_, err := w.Write([]byte("Ok\n"))
 	if err != nil {
 		errorChannel <- err
 
 		return
 	}
 
+	ticker := time.NewTicker(interval)
+
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				startTime := time.Now()
+
+				rebuildIndex(args, index, formats, encoder, errorChannel)
+
 				if Verbose {
-			fmt.Printf("%s | SERVE: Index rebuild requested by %s took %s\n",
+					fmt.Printf("%s | INDEX: Automatic rebuild took %s\n",
 						startTime.Format(logDate),
-				realIP(r),
 						time.Since(startTime).Round(time.Microsecond),
 					)
 				}
+			case <-quit:
+				ticker.Stop()
+
+				return
 			}
 		}
+	}()
+}
+
+func rebuildIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
+	index.clear()
+
+	fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
+}
 
 func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
 	if IndexFile != "" {

cmd/info.go

@@ -13,9 +13,40 @@ import (
 	"time"
 
 	"github.com/julienschmidt/httprouter"
+	"github.com/klauspost/compress/zstd"
 	"seedno.de/seednode/roulette/types"
 )
 
+func serveExtensions(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle {
+	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		startTime := time.Now()
+
+		w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
+
+		var extensions string
+
+		if available {
+			extensions = types.SupportedFormats.GetExtensions()
+		} else {
+			extensions = formats.GetExtensions()
+		}
+
+		written, err := w.Write([]byte(extensions))
+		if err != nil {
+			errorChannel <- err
+		}
+
+		if Verbose {
+			fmt.Printf("%s | SERVE: Registered extension list (%s) to %s in %s\n",
+				startTime.Format(logDate),
+				humanReadableSize(written),
+				realIP(r),
+				time.Since(startTime).Round(time.Microsecond),
+			)
+		}
+	}
+}
+
 func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		startTime := time.Now()
@@ -55,29 +86,24 @@ func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) http
 	}
 }
 
-func serveExtensions(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle {
+func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		startTime := time.Now()
 
+		rebuildIndex(args, index, formats, encoder, errorChannel)
+
 		w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
 
-		var extensions string
+		_, err := w.Write([]byte("Ok\n"))
-
-		if available {
-			extensions = types.SupportedFormats.GetExtensions()
-		} else {
-			extensions = formats.GetExtensions()
-		}
-
-		written, err := w.Write([]byte(extensions))
 		if err != nil {
 			errorChannel <- err
+
+			return
 		}
 
 		if Verbose {
-			fmt.Printf("%s | SERVE: Registered extension list (%s) to %s in %s\n",
+			fmt.Printf("%s | SERVE: Index rebuild requested by %s took %s\n",
 				startTime.Format(logDate),
-				humanReadableSize(written),
 				realIP(r),
 				time.Since(startTime).Round(time.Microsecond),
 			)
@@ -115,9 +141,10 @@ func serveMediaTypes(formats types.Types, available bool, errorChannel chan<- er
 	}
 }
 
-func registerInfoHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
+func registerAPIHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
 	if Index {
 		mux.GET(Prefix+AdminPrefix+"/index", serveIndex(args, index, errorChannel))
+		mux.POST(Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel))
 	}
 
 	mux.GET(Prefix+AdminPrefix+"/extensions/available", serveExtensions(formats, true, errorChannel))

cmd/root.go

@@ -17,13 +17,14 @@ import (
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "6.3.1"
+	ReleaseVersion    string = "7.0.0"
 )
 
 var (
 	AdminPrefix     string
 	All             bool
 	AllowEmpty      bool
+	API             bool
 	Audio           bool
 	BinaryPrefix    bool
 	Bind            string
@@ -38,12 +39,11 @@ var (
 	Filtering       bool
 	Flash           bool
 	Fun             bool
-	Ignore          bool
+	Ignore          string
-	IgnoreFile      string
 	Images          bool
 	Index           bool
 	IndexFile       string
-	Info            bool
+	IndexInterval   string
 	MaxFileCount    int
 	MinFileCount    int
 	Port            int
@@ -83,7 +83,7 @@ var (
 				return ErrInvalidPort
 			case Concurrency < 1:
 				return ErrInvalidConcurrency
-			case Ignore && !regexp.MustCompile(AllowedCharacters).MatchString(IgnoreFile):
+			case Ignore != "" && !regexp.MustCompile(AllowedCharacters).MatchString(Ignore):
 				return ErrInvalidIgnoreFile
 			case AdminPrefix != "" && !regexp.MustCompile(AllowedCharacters).MatchString(AdminPrefix):
 				return ErrInvalidAdminPrefix
@@ -117,6 +117,7 @@ func init() {
 	rootCmd.Flags().StringVar(&AdminPrefix, "admin-prefix", "", "string to prepend to administrative paths")
 	rootCmd.Flags().BoolVarP(&All, "all", "a", false, "enable all supported file types")
 	rootCmd.Flags().BoolVar(&AllowEmpty, "allow-empty", false, "allow specifying paths containing no supported files")
+	rootCmd.Flags().BoolVar(&API, "api", false, "expose REST API")
 	rootCmd.Flags().BoolVar(&Audio, "audio", false, "enable support for audio files")
 	rootCmd.Flags().BoolVar(&BinaryPrefix, "binary-prefix", false, "use IEC binary prefixes instead of SI decimal prefixes")
 	rootCmd.Flags().StringVarP(&Bind, "bind", "b", "0.0.0.0", "address to bind to")
@@ -131,12 +132,11 @@ func init() {
 	rootCmd.Flags().BoolVarP(&Filtering, "filter", "f", false, "enable filtering")
 	rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)")
 	rootCmd.Flags().BoolVar(&Fun, "fun", false, "add a bit of excitement to your day")
-	rootCmd.Flags().BoolVar(&Ignore, "ignore", false, "skip all directories containing a specified filename")
+	rootCmd.Flags().StringVar(&Ignore, "ignore", "", "filename used to indicate directory should be skipped")
-	rootCmd.Flags().StringVar(&IgnoreFile, "ignore-file", ".roulette-ignore", "filename used to indicate directory should be skipped")
 	rootCmd.Flags().BoolVar(&Images, "images", false, "enable support for image files")
 	rootCmd.Flags().BoolVar(&Index, "index", false, "generate index of supported file paths at startup")
 	rootCmd.Flags().StringVar(&IndexFile, "index-file", "", "path to optional persistent index file")
-	rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints")
+	rootCmd.Flags().StringVar(&IndexInterval, "index-interval", "", "interval at which to regenerate index (e.g. \"5m\" or \"1h\")")
 	rootCmd.Flags().IntVar(&MaxFileCount, "max-file-count", math.MaxInt32, "skip directories with file counts above this value")
 	rootCmd.Flags().IntVar(&MinFileCount, "min-file-count", 0, "skip directories with file counts below this value")
 	rootCmd.Flags().IntVarP(&Port, "port", "p", 8080, "port to listen on")

cmd/web.go

@@ -52,21 +52,6 @@ func newPage(title, body string) string {
 	return htmlBody.String()
 }
 
-func noFiles(w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-
-	w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
-
-	w.Write([]byte("No files found in the specified path(s).\n"))
-
-	if Verbose {
-		fmt.Printf("%s | SERVE: Empty path notification to %s\n",
-			startTime.Format(logDate),
-			r.RemoteAddr,
-		)
-	}
-}
-
 func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		prefix := Prefix + sourcePrefix
@@ -221,7 +206,18 @@ func serveRoot(paths []string, index *fileIndex, filename *regexp.Regexp, format
 			path, err = newFile(list, sortOrder, filename, formats)
 			switch {
 			case path == "":
-				noFiles(w, r)
+				startTime := time.Now()
+
+				w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
+
+				w.Write([]byte("No files found in the specified path(s).\n"))
+
+				if Verbose {
+					fmt.Printf("%s | SERVE: Empty path notification to %s\n",
+						startTime.Format(logDate),
+						r.RemoteAddr,
+					)
+				}
 
 				return
 			case err != nil && err == ErrNoMediaFound:
@@ -458,9 +454,10 @@ func redirectRoot() httprouter.Handle {
 }
 
 func ServePage(args []string) error {
+	var err error
+
 	timeZone := os.Getenv("TZ")
 	if timeZone != "" {
-		var err error
 		time.Local, err = time.LoadLocation(timeZone)
 		if err != nil {
 			return err
@@ -584,14 +581,19 @@ func ServePage(args []string) error {
 
 	mux.GET(Prefix+"/version", serveVersion(errorChannel))
 
-	if Index {
+	quit := make(chan struct{})
-		mux.GET(Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, encoder, errorChannel))
+	defer close(quit)
 
-		importIndex(paths, index, formats, encoder, errorChannel)
+	if API {
+		registerAPIHandlers(mux, args, index, formats, encoder, errorChannel)
 	}
 
-	if Info {
+	if Index {
-		registerInfoHandlers(mux, args, index, formats, errorChannel)
+		importIndex(paths, index, formats, encoder, errorChannel)
+
+		if IndexInterval != "" {
+			registerIndexInterval(args, index, formats, encoder, quit, errorChannel)
+		}
 	}
 
 	if Profile {