Add flags to modify max concurrency directory and file scans

README.md

@@ -127,8 +127,10 @@ Flags:
   -h, --help                       help for roulette
       --images                     enable support for image files
   -i, --info                       expose informational endpoints
-      --maximum-files uint   skip directories with file counts above this value (default 4294967295)
-      --minimum-files uint   skip directories with file counts below this value (default 1)
+      --max-directory-scans uint   number of directories to scan at once (default 32)
+      --max-file-count uint        skip directories with file counts above this value (default 4294967295)
+      --max-file-scans uint        number of files to scan at once (default 256)
+      --min-file-count uint        skip directories with file counts below this value (default 1)
       --page-length uint32         pagination length for info pages
   -p, --port uint16                port to listen on (default 8080)
       --prefix string              root path for http handlers (for reverse proxying) (default "/")

cmd/files.go

@@ -21,14 +21,6 @@ import (
 	"seedno.de/seednode/roulette/types"
 )
 
-type maxConcurrency int
-
-const (
-	// avoid hitting default open file descriptor limits (1024)
-	maxDirectoryScans maxConcurrency = 32
-	maxFileScans      maxConcurrency = 256
-)
-
 type regexes struct {
 	alphanumeric *regexp.Regexp
 	filename     *regexp.Regexp
@@ -331,7 +323,7 @@ func walkPath(path string, fileChannel chan<- string, fileScans chan int, stats
 				errorChannel <- err
 			}
 
-			if files > 0 && (files < int(MinimumFileCount)) || (files > int(MaximumFileCount)) {
+			if files > 0 && (files < int(MinFileCount)) || (files > int(MaxFileCount)) {
 				// This count will not otherwise include the parent directory itself, so increment by one
 				stats.directoriesSkipped <- directories + 1
 				stats.filesSkipped <- files
@@ -368,8 +360,8 @@ func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Typ
 
 	fileChannel := make(chan string)
 	errorChannel := make(chan error)
-	directoryScans := make(chan int, maxDirectoryScans)
-	fileScans := make(chan int, maxFileScans)
+	directoryScans := make(chan int, MaxDirScans)
+	fileScans := make(chan int, MaxFileScans)
 	done := make(chan bool, 1)
 
 	stats := &scanStats{

cmd/info.go

@@ -241,7 +241,7 @@ func serveAvailableMimeTypes() httprouter.Handle {
 		w.Write(response)
 
 		if Verbose {
-			fmt.Printf("%s | Served available MIME type list (%s) to %s in %s\n",
+			fmt.Printf("%s | Serve: Available MIME type list (%s) to %s in %s\n",
 				startTime.Format(logDate),
 				humanReadableSize(len(response)),
 				realIP(r),
@@ -262,7 +262,7 @@ func serveEnabledMimeTypes(formats *types.Types) httprouter.Handle {
 		w.Write(response)
 
 		if Verbose {
-			fmt.Printf("%s | Served registered MIME type list (%s) to %s in %s\n",
+			fmt.Printf("%s | Serve: Registered MIME type list (%s) to %s in %s\n",
 				startTime.Format(logDate),
 				humanReadableSize(len(response)),
 				realIP(r),

cmd/root.go

@@ -5,13 +5,14 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 package cmd
 
 import (
+	"errors"
 	"log"
 
 	"github.com/spf13/cobra"
 )
 
 const (
-	ReleaseVersion string = "0.92.2"
+	ReleaseVersion string = "0.93.0"
 )
 
 var (
@@ -29,8 +30,10 @@ var (
 	Handlers      bool
 	Images        bool
 	Info          bool
-	MaximumFileCount uint
-	MinimumFileCount uint
+	MaxDirScans   uint
+	MaxFileScans  uint
+	MaxFileCount  uint
+	MinFileCount  uint
 	PageLength    uint32
 	Port          uint16
 	Prefix        string
@@ -48,6 +51,17 @@ var (
 		Use:   "roulette <path> [path]...",
 		Short: "Serves random media from the specified directories.",
 		Args:  cobra.MinimumNArgs(1),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if MaxDirScans < 1 {
+				return errors.New("max directory scan count must be a positive integer")
+			}
+
+			if MaxFileScans < 1 {
+				return errors.New("max file scan count must be a positive integer")
+			}
+
+			return nil
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			err := ServePage(args)
 			if err != nil {
@@ -81,8 +95,10 @@ func init() {
 	rootCmd.Flags().BoolVar(&Handlers, "handlers", false, "display registered handlers (for debugging)")
 	rootCmd.Flags().BoolVar(&Images, "images", false, "enable support for image files")
 	rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints")
-	rootCmd.Flags().UintVar(&MaximumFileCount, "maximum-files", 1<<32-1, "skip directories with file counts above this value")
-	rootCmd.Flags().UintVar(&MinimumFileCount, "minimum-files", 1, "skip directories with file counts below this value")
+	rootCmd.Flags().UintVar(&MaxDirScans, "max-directory-scans", 32, "number of directories to scan at once")
+	rootCmd.Flags().UintVar(&MaxFileScans, "max-file-scans", 256, "number of files to scan at once")
+	rootCmd.Flags().UintVar(&MaxFileCount, "max-file-count", 1<<32-1, "skip directories with file counts above this value")
+	rootCmd.Flags().UintVar(&MinFileCount, "min-file-count", 1, "skip directories with file counts below this value")
 	rootCmd.Flags().Uint32Var(&PageLength, "page-length", 0, "pagination length for info pages")
 	rootCmd.Flags().Uint16VarP(&Port, "port", "p", 8080, "port to listen on")
 	rootCmd.Flags().StringVar(&Prefix, "prefix", "/", "root path for http handlers (for reverse proxying)")