diff --git a/README.md b/README.md index 8babbcf..bd8941f 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,18 @@ x86_64 and ARM Docker images of latest version: `oci.seedno.de/seednode/roulette Dockerfile available [here](https://git.seedno.de/seednode/roulette/raw/branch/master/docker/Dockerfile). +## Caching + +If the `-c|--cache` flag is passed, the indices of all specified paths will be cached on start. + +This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests. + +If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request. + +The cache can be regenerated at any time by accessing the `/clear_cache` endpoint. + +If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated. + ## Filtering You can provide a comma-delimited string of alphanumeric patterns to match via the `include=` query parameter, assuming the `-f|--filter` flag is enabled. @@ -32,6 +44,39 @@ You can also combine these two parameters, with exclusions taking priority over Both filtering parameters ignore the file extension and full path; they only compare against the bare filename. +## Info + +If the `-i|--info` flag is passed, six additional endpoints are registered. + +The first of these—`/html` and `/json`—return the contents of the index, in HTML and JSON formats respectively. + +If `--page-length` is also set, these can be viewed in paginated form by appending `/n`, e.g. `/html/5` for the fifth page. + +This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index. + +The remaining four endpoints—`/available_extensions`, `/enabled_extensions`, `/available_mime_types` and `/enabled_mime_types`—return information about the registered file types. + +## Refresh + +If a positive-value `refresh=` query parameter is provided, the page will reload after that interval. + +This can be used to generate a sort of slideshow of files. + +Minimum accepted value is 500ms, as anything lower seems to cause inconsistent behavior. This might be changed in a future release. + +Supported units are `ns`, `us`/`µs`, `ms`, `s`, `m`, and `h`. + +## Russian +If the `--russian` flag is passed, everything functions exactly as you would expect. + +That is, files will be deleted after being served. This is not a joke, you *will* lose data. + +This uses `os.Remove()` and checks to ensure the specified file is inside one of the paths passed to `roulette`. + +That said, this has not been tested to any real extent, so only pass this flag on systems you don't care about. + +Enjoy! + ## Sorting You can specify a sorting pattern via the `sort=` query parameter, assuming the `-s|--sort` flag is enabled. @@ -54,59 +99,6 @@ If any other (or no) value is provided, the selected file will be random. Note: These patterns require sequentially-numbered files matching the following pattern: `filename###.extension`. -## Refresh - -If a positive-value `refresh=` query parameter is provided, the page will reload after that interval. - -This can be used to generate a sort of slideshow of files. - -Minimum accepted value is 500ms, as anything lower seems to cause inconsistent behavior. This might be changed in a future release. - -Supported units are `ns`, `us`/`µs`, `ms`, `s`, `m`, and `h`. - -## Caching - -If the `-c|--cache` flag is passed, the indices of all specified paths will be cached on start. - -This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests. - -If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request. - -The cache can be regenerated at any time by accessing the `/clear_cache` endpoint. - -If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated. - -## Info - -If the `-i|--info` flag is passed, six additional endpoints are registered. - -The first of these—`/html` and `/json`—return the contents of the index, in HTML and JSON formats respectively. - -If `--page-length` is also set, these can be viewed in paginated form by appending `/n`, e.g. `/html/5` for the fifth page. - -This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index. - -The remaining four endpoints—`/available_extensions`, `/enabled_extensions`, `/available_mime_types` and `/enabled_mime_types`—return information about the registered file types. - -## Statistics - -If the `--stats` flag is passed, an additional endpoint, `/stats`, is registered. - -When accessed, this endpoint returns a JSON document listing every file served, along with the number of times it has been served, its filesize, and timestamps of when it was served. - -If `--page-length` is also set, this can be viewed in paginated form by appending `/n`, e.g. `/stats/5` for the fifth page. - -## Russian -If the `--russian` flag is passed, everything functions exactly as you would expect. - -That is, files will be deleted after being served. This is not a joke, you *will* lose data. - -This uses `os.Remove()` and checks to ensure the specified file is inside one of the paths passed to `roulette`. - -That said, this has not been tested to any real extent, so only pass this flag on systems you don't care about. - -Enjoy! - ## Usage output ``` Serves random media from the specified directories. @@ -134,8 +126,6 @@ Flags: --refresh-interval string force refresh interval equal to this duration (minimum 500ms) --russian remove selected images after serving -s, --sort enable sorting - --stats expose stats endpoint - --stats-file string path to optional persistent stats file --text enable support for text files -v, --verbose log accessed files and other information to stdout -V, --version display version and exit @@ -145,4 +135,4 @@ Flags: ## Building the Docker container From inside the `docker/` subdirectory, build the image using the following command: -`REGISTRY= LATEST=yes TAG=alpine ./build.sh` +`REGISTRY= LATEST=yes TAG=alpine ./build.sh` \ No newline at end of file diff --git a/cmd/files.go b/cmd/files.go index c88f4b8..ff646ce 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -73,15 +73,6 @@ func (splitPath *splitPath) decrement() { splitPath.number = splitPath.number - 1 } -func contains(slice []string, value string) bool { - for _, v := range slice { - if v == value { - return true - } - } - return false -} - func humanReadableSize(bytes int) string { const unit = 1000 diff --git a/cmd/root.go b/cmd/root.go index b4c4117..8ad7a43 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -12,7 +12,7 @@ import ( ) const ( - ReleaseVersion string = "0.78.2" + ReleaseVersion string = "0.79.0" ) var ( @@ -34,8 +34,6 @@ var ( RefreshInterval string Russian bool Sorting bool - Statistics bool - StatisticsFile string Text bool Verbose bool Version bool @@ -92,8 +90,6 @@ func init() { rootCmd.Flags().StringVar(&RefreshInterval, "refresh-interval", "", "force refresh interval equal to this duration (minimum 500ms)") rootCmd.Flags().BoolVar(&Russian, "russian", false, "remove selected images after serving") rootCmd.Flags().BoolVarP(&Sorting, "sort", "s", false, "enable sorting") - rootCmd.Flags().BoolVar(&Statistics, "stats", false, "expose stats endpoint") - rootCmd.Flags().StringVar(&StatisticsFile, "stats-file", "", "path to optional persistent stats file") rootCmd.Flags().BoolVar(&Text, "text", false, "enable support for text files") rootCmd.Flags().BoolVarP(&Verbose, "verbose", "v", false, "log accessed files and other information to stdout") rootCmd.Flags().BoolVarP(&Version, "version", "V", false, "display version and exit") diff --git a/cmd/stats.go b/cmd/stats.go deleted file mode 100644 index 8b001e4..0000000 --- a/cmd/stats.go +++ /dev/null @@ -1,253 +0,0 @@ -/* -Copyright © 2023 Seednode -*/ - -package cmd - -import ( - "encoding/gob" - "encoding/json" - "fmt" - "net/http" - "os" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/julienschmidt/httprouter" - "github.com/klauspost/compress/zstd" -) - -type serveStats struct { - mutex sync.RWMutex - list []string - count map[string]uint32 - size map[string]string - times map[string][]string -} - -type publicServeStats struct { - List []string - Count map[string]uint32 - Size map[string]string - Times map[string][]string -} - -type timesServed struct { - File string - Served uint32 - Size string - Times []string -} - -func (stats *serveStats) incrementCounter(file string, timestamp time.Time, filesize string) { - stats.mutex.Lock() - - stats.count[file]++ - - stats.times[file] = append(stats.times[file], timestamp.Format(logDate)) - - _, exists := stats.size[file] - if !exists { - stats.size[file] = filesize - } - - if !contains(stats.list, file) { - stats.list = append(stats.list, file) - } - - stats.mutex.Unlock() -} - -func (stats *serveStats) Import(source *publicServeStats) { - stats.mutex.Lock() - - copy(stats.list, source.List) - - for k, v := range source.Count { - fmt.Printf("Setting count[%s] to %d\n", k, v) - stats.count[k] = v - } - - for k, v := range source.Size { - fmt.Printf("Setting size[%s] to %v\n", k, v) - - stats.size[k] = v - } - - for k, v := range source.Times { - fmt.Printf("Setting times[%s] to %v\n", k, v) - stats.times[k] = v - } - - stats.mutex.Unlock() -} - -func (source *serveStats) Export() *publicServeStats { - source.mutex.RLock() - - stats := &publicServeStats{ - List: make([]string, len(source.list)), - Count: make(map[string]uint32, len(source.count)), - Size: make(map[string]string, len(source.size)), - Times: make(map[string][]string, len(source.times)), - } - - copy(stats.List, source.list) - - for k, v := range source.count { - stats.Count[k] = v - } - - for k, v := range source.size { - stats.Size[k] = v - } - - for k, v := range source.times { - stats.Times[k] = v - } - - source.mutex.RUnlock() - - return stats -} - -func (stats *serveStats) exportFile(path string) error { - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) - if err != nil { - return err - } - defer file.Close() - - z, err := zstd.NewWriter(file) - if err != nil { - return err - } - defer z.Close() - - enc := gob.NewEncoder(z) - - err = enc.Encode(stats.Export()) - if err != nil { - return err - } - - return nil -} - -func (stats *serveStats) importFile(path string) error { - file, err := os.OpenFile(path, os.O_RDONLY, 0600) - if err != nil { - return err - } - defer file.Close() - - z, err := zstd.NewReader(file) - if err != nil { - return err - } - defer z.Close() - - dec := gob.NewDecoder(z) - - source := &publicServeStats{ - List: []string{}, - Count: make(map[string]uint32), - Size: make(map[string]string), - Times: make(map[string][]string), - } - - err = dec.Decode(source) - if err != nil { - return err - } - - stats.Import(source) - - return nil -} - -func (source *serveStats) listFiles(page int) ([]byte, error) { - stats := source.Export() - - sort.SliceStable(stats.List, func(p, q int) bool { - return strings.ToLower(stats.List[p]) < strings.ToLower(stats.List[q]) - }) - - var startIndex, stopIndex int - - if page == -1 { - startIndex = 0 - stopIndex = len(stats.List) - } else { - startIndex = ((page - 1) * int(PageLength)) - stopIndex = (startIndex + int(PageLength)) - } - - if startIndex > len(stats.List)-1 { - return []byte("{}"), nil - } - - if stopIndex > len(stats.List) { - stopIndex = len(stats.List) - } - - a := make([]timesServed, (stopIndex - startIndex)) - - for k, v := range stats.List[startIndex:stopIndex] { - a[k] = timesServed{v, stats.Count[v], stats.Size[v], stats.Times[v]} - } - - r, err := json.MarshalIndent(a, "", " ") - if err != nil { - return []byte{}, err - } - - return r, nil -} - -func serveStatsPage(args []string, stats *serveStats) httprouter.Handle { - return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { - startTime := time.Now() - - page, err := strconv.Atoi(p.ByName("page")) - if err != nil || page == 0 { - page = -1 - } - - response, err := stats.listFiles(page) - if err != nil { - fmt.Println(err) - - serverError(w, r, nil) - - return - } - - w.Header().Set("Content-Type", "application/json") - - w.Write(response) - - if Verbose { - fmt.Printf("%s | Served statistics page (%s) to %s in %s\n", - startTime.Format(logDate), - humanReadableSize(len(response)), - realIP(r), - time.Since(startTime).Round(time.Microsecond), - ) - } - - if StatisticsFile != "" { - stats.exportFile(StatisticsFile) - } - } -} - -func registerStatsHandlers(mux *httprouter.Router, args []string, stats *serveStats) { - mux.GET("/stats", serveStatsPage(args, stats)) - if PageLength != 0 { - mux.GET("/stats/:page", serveStatsPage(args, stats)) - } -} diff --git a/cmd/web.go b/cmd/web.go index 68574b4..dd6beed 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -12,14 +12,12 @@ import ( "net" "net/http" "os" - "os/signal" "path/filepath" "regexp" "runtime" "strconv" "strings" "sync" - "syscall" "time" "github.com/julienschmidt/httprouter" @@ -40,7 +38,7 @@ const ( timeout time.Duration = 10 * time.Second ) -func serveStaticFile(paths []string, stats *serveStats, cache *fileCache) httprouter.Handle { +func serveStaticFile(paths []string, cache *fileCache) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { path := strings.TrimPrefix(r.URL.Path, sourcePrefix) @@ -122,11 +120,6 @@ func serveStaticFile(paths []string, stats *serveStats, cache *fileCache) httpro time.Since(startTime).Round(time.Microsecond), ) } - - if Statistics { - stats.incrementCounter(filePath, startTime, fileSize) - } - } } @@ -368,14 +361,6 @@ func ServePage(args []string) error { WriteTimeout: 5 * time.Minute, } - stats := &serveStats{ - mutex: sync.RWMutex{}, - list: []string{}, - count: make(map[string]uint32), - size: make(map[string]string), - times: make(map[string][]string), - } - mux.PanicHandler = serverErrorHandler() mux.GET("/", serveRoot(paths, regexes, cache, formats)) @@ -386,7 +371,7 @@ func ServePage(args []string) error { mux.GET(mediaPrefix+"/*media", serveMedia(paths, regexes, formats)) - mux.GET(sourcePrefix+"/*static", serveStaticFile(paths, stats, cache)) + mux.GET(sourcePrefix+"/*static", serveStaticFile(paths, cache)) mux.GET("/version", serveVersion()) @@ -406,33 +391,6 @@ func ServePage(args []string) error { fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n") } - if Statistics { - if StatisticsFile != "" { - StatisticsFile, err = normalizePath(StatisticsFile) - if err != nil { - return err - } - - err := stats.importFile(StatisticsFile) - if err != nil { - return err - } - - gracefulShutdown := make(chan os.Signal, 1) - signal.Notify(gracefulShutdown, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-gracefulShutdown - - stats.exportFile(StatisticsFile) - - os.Exit(0) - }() - } - - registerStatsHandlers(mux, args, stats) - } - err = srv.ListenAndServe() if !errors.Is(err, http.ErrServerClosed) { return err