Removed statistics (sorry) pending full rewrite
This commit is contained in:
Seednode
parent
d2d9418b8b
commit
ce99374a9f
100
README.md
100
README.md
|
|
@ -18,6 +18,18 @@ x86_64 and ARM Docker images of latest version: `oci.seedno.de/seednode/roulette
|
|||
|
||||
Dockerfile available [here](https://git.seedno.de/seednode/roulette/raw/branch/master/docker/Dockerfile).
|
||||
|
||||
## Caching
|
||||
|
||||
If the `-c|--cache` flag is passed, the indices of all specified paths will be cached on start.
|
||||
|
||||
This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
|
||||
|
||||
If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request.
|
||||
|
||||
The cache can be regenerated at any time by accessing the `/clear_cache` endpoint.
|
||||
|
||||
If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated.
|
||||
|
||||
## Filtering
|
||||
|
||||
You can provide a comma-delimited string of alphanumeric patterns to match via the `include=` query parameter, assuming the `-f|--filter` flag is enabled.
|
||||
|
|
@ -32,6 +44,39 @@ You can also combine these two parameters, with exclusions taking priority over
|
|||
|
||||
Both filtering parameters ignore the file extension and full path; they only compare against the bare filename.
|
||||
|
||||
## Info
|
||||
|
||||
If the `-i|--info` flag is passed, six additional endpoints are registered.
|
||||
|
||||
The first of these—`/html` and `/json`—return the contents of the index, in HTML and JSON formats respectively.
|
||||
|
||||
If `--page-length` is also set, these can be viewed in paginated form by appending `/n`, e.g. `/html/5` for the fifth page.
|
||||
|
||||
This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index.
|
||||
|
||||
The remaining four endpoints—`/available_extensions`, `/enabled_extensions`, `/available_mime_types` and `/enabled_mime_types`—return information about the registered file types.
|
||||
|
||||
## Refresh
|
||||
|
||||
If a positive-value `refresh=<integer><unit>` query parameter is provided, the page will reload after that interval.
|
||||
|
||||
This can be used to generate a sort of slideshow of files.
|
||||
|
||||
Minimum accepted value is 500ms, as anything lower seems to cause inconsistent behavior. This might be changed in a future release.
|
||||
|
||||
Supported units are `ns`, `us`/`µs`, `ms`, `s`, `m`, and `h`.
|
||||
|
||||
## Russian
|
||||
If the `--russian` flag is passed, everything functions exactly as you would expect.
|
||||
|
||||
That is, files will be deleted after being served. This is not a joke, you *will* lose data.
|
||||
|
||||
This uses `os.Remove()` and checks to ensure the specified file is inside one of the paths passed to `roulette`.
|
||||
|
||||
That said, this has not been tested to any real extent, so only pass this flag on systems you don't care about.
|
||||
|
||||
Enjoy!
|
||||
|
||||
## Sorting
|
||||
|
||||
You can specify a sorting pattern via the `sort=` query parameter, assuming the `-s|--sort` flag is enabled.
|
||||
|
|
@ -54,59 +99,6 @@ If any other (or no) value is provided, the selected file will be random.
|
|||
|
||||
Note: These patterns require sequentially-numbered files matching the following pattern: `filename###.extension`.
|
||||
|
||||
## Refresh
|
||||
|
||||
If a positive-value `refresh=<integer><unit>` query parameter is provided, the page will reload after that interval.
|
||||
|
||||
This can be used to generate a sort of slideshow of files.
|
||||
|
||||
Minimum accepted value is 500ms, as anything lower seems to cause inconsistent behavior. This might be changed in a future release.
|
||||
|
||||
Supported units are `ns`, `us`/`µs`, `ms`, `s`, `m`, and `h`.
|
||||
|
||||
## Caching
|
||||
|
||||
If the `-c|--cache` flag is passed, the indices of all specified paths will be cached on start.
|
||||
|
||||
This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
|
||||
|
||||
If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request.
|
||||
|
||||
The cache can be regenerated at any time by accessing the `/clear_cache` endpoint.
|
||||
|
||||
If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated.
|
||||
|
||||
## Info
|
||||
|
||||
If the `-i|--info` flag is passed, six additional endpoints are registered.
|
||||
|
||||
The first of these—`/html` and `/json`—return the contents of the index, in HTML and JSON formats respectively.
|
||||
|
||||
If `--page-length` is also set, these can be viewed in paginated form by appending `/n`, e.g. `/html/5` for the fifth page.
|
||||
|
||||
This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index.
|
||||
|
||||
The remaining four endpoints—`/available_extensions`, `/enabled_extensions`, `/available_mime_types` and `/enabled_mime_types`—return information about the registered file types.
|
||||
|
||||
## Statistics
|
||||
|
||||
If the `--stats` flag is passed, an additional endpoint, `/stats`, is registered.
|
||||
|
||||
When accessed, this endpoint returns a JSON document listing every file served, along with the number of times it has been served, its filesize, and timestamps of when it was served.
|
||||
|
||||
If `--page-length` is also set, this can be viewed in paginated form by appending `/n`, e.g. `/stats/5` for the fifth page.
|
||||
|
||||
## Russian
|
||||
If the `--russian` flag is passed, everything functions exactly as you would expect.
|
||||
|
||||
That is, files will be deleted after being served. This is not a joke, you *will* lose data.
|
||||
|
||||
This uses `os.Remove()` and checks to ensure the specified file is inside one of the paths passed to `roulette`.
|
||||
|
||||
That said, this has not been tested to any real extent, so only pass this flag on systems you don't care about.
|
||||
|
||||
Enjoy!
|
||||
|
||||
## Usage output
|
||||
```
|
||||
Serves random media from the specified directories.
|
||||
|
|
@ -134,8 +126,6 @@ Flags:
|
|||
--refresh-interval string force refresh interval equal to this duration (minimum 500ms)
|
||||
--russian remove selected images after serving
|
||||
-s, --sort enable sorting
|
||||
--stats expose stats endpoint
|
||||
--stats-file string path to optional persistent stats file
|
||||
--text enable support for text files
|
||||
-v, --verbose log accessed files and other information to stdout
|
||||
-V, --version display version and exit
|
||||
|
|
|
|||
|
|
@ -73,15 +73,6 @@ func (splitPath *splitPath) decrement() {
|
|||
splitPath.number = splitPath.number - 1
|
||||
}
|
||||
|
||||
func contains(slice []string, value string) bool {
|
||||
for _, v := range slice {
|
||||
if v == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func humanReadableSize(bytes int) string {
|
||||
const unit = 1000
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
ReleaseVersion string = "0.78.2"
|
||||
ReleaseVersion string = "0.79.0"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -34,8 +34,6 @@ var (
|
|||
RefreshInterval string
|
||||
Russian bool
|
||||
Sorting bool
|
||||
Statistics bool
|
||||
StatisticsFile string
|
||||
Text bool
|
||||
Verbose bool
|
||||
Version bool
|
||||
|
|
@ -92,8 +90,6 @@ func init() {
|
|||
rootCmd.Flags().StringVar(&RefreshInterval, "refresh-interval", "", "force refresh interval equal to this duration (minimum 500ms)")
|
||||
rootCmd.Flags().BoolVar(&Russian, "russian", false, "remove selected images after serving")
|
||||
rootCmd.Flags().BoolVarP(&Sorting, "sort", "s", false, "enable sorting")
|
||||
rootCmd.Flags().BoolVar(&Statistics, "stats", false, "expose stats endpoint")
|
||||
rootCmd.Flags().StringVar(&StatisticsFile, "stats-file", "", "path to optional persistent stats file")
|
||||
rootCmd.Flags().BoolVar(&Text, "text", false, "enable support for text files")
|
||||
rootCmd.Flags().BoolVarP(&Verbose, "verbose", "v", false, "log accessed files and other information to stdout")
|
||||
rootCmd.Flags().BoolVarP(&Version, "version", "V", false, "display version and exit")
|
||||
|
|
|
|||
253
cmd/stats.go
253
cmd/stats.go
|
|
@ -1,253 +0,0 @@
|
|||
/*
|
||||
Copyright © 2023 Seednode <seednode@seedno.de>
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
type serveStats struct {
|
||||
mutex sync.RWMutex
|
||||
list []string
|
||||
count map[string]uint32
|
||||
size map[string]string
|
||||
times map[string][]string
|
||||
}
|
||||
|
||||
type publicServeStats struct {
|
||||
List []string
|
||||
Count map[string]uint32
|
||||
Size map[string]string
|
||||
Times map[string][]string
|
||||
}
|
||||
|
||||
type timesServed struct {
|
||||
File string
|
||||
Served uint32
|
||||
Size string
|
||||
Times []string
|
||||
}
|
||||
|
||||
func (stats *serveStats) incrementCounter(file string, timestamp time.Time, filesize string) {
|
||||
stats.mutex.Lock()
|
||||
|
||||
stats.count[file]++
|
||||
|
||||
stats.times[file] = append(stats.times[file], timestamp.Format(logDate))
|
||||
|
||||
_, exists := stats.size[file]
|
||||
if !exists {
|
||||
stats.size[file] = filesize
|
||||
}
|
||||
|
||||
if !contains(stats.list, file) {
|
||||
stats.list = append(stats.list, file)
|
||||
}
|
||||
|
||||
stats.mutex.Unlock()
|
||||
}
|
||||
|
||||
func (stats *serveStats) Import(source *publicServeStats) {
|
||||
stats.mutex.Lock()
|
||||
|
||||
copy(stats.list, source.List)
|
||||
|
||||
for k, v := range source.Count {
|
||||
fmt.Printf("Setting count[%s] to %d\n", k, v)
|
||||
stats.count[k] = v
|
||||
}
|
||||
|
||||
for k, v := range source.Size {
|
||||
fmt.Printf("Setting size[%s] to %v\n", k, v)
|
||||
|
||||
stats.size[k] = v
|
||||
}
|
||||
|
||||
for k, v := range source.Times {
|
||||
fmt.Printf("Setting times[%s] to %v\n", k, v)
|
||||
stats.times[k] = v
|
||||
}
|
||||
|
||||
stats.mutex.Unlock()
|
||||
}
|
||||
|
||||
func (source *serveStats) Export() *publicServeStats {
|
||||
source.mutex.RLock()
|
||||
|
||||
stats := &publicServeStats{
|
||||
List: make([]string, len(source.list)),
|
||||
Count: make(map[string]uint32, len(source.count)),
|
||||
Size: make(map[string]string, len(source.size)),
|
||||
Times: make(map[string][]string, len(source.times)),
|
||||
}
|
||||
|
||||
copy(stats.List, source.list)
|
||||
|
||||
for k, v := range source.count {
|
||||
stats.Count[k] = v
|
||||
}
|
||||
|
||||
for k, v := range source.size {
|
||||
stats.Size[k] = v
|
||||
}
|
||||
|
||||
for k, v := range source.times {
|
||||
stats.Times[k] = v
|
||||
}
|
||||
|
||||
source.mutex.RUnlock()
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
func (stats *serveStats) exportFile(path string) error {
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
z, err := zstd.NewWriter(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer z.Close()
|
||||
|
||||
enc := gob.NewEncoder(z)
|
||||
|
||||
err = enc.Encode(stats.Export())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (stats *serveStats) importFile(path string) error {
|
||||
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
z, err := zstd.NewReader(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer z.Close()
|
||||
|
||||
dec := gob.NewDecoder(z)
|
||||
|
||||
source := &publicServeStats{
|
||||
List: []string{},
|
||||
Count: make(map[string]uint32),
|
||||
Size: make(map[string]string),
|
||||
Times: make(map[string][]string),
|
||||
}
|
||||
|
||||
err = dec.Decode(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stats.Import(source)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (source *serveStats) listFiles(page int) ([]byte, error) {
|
||||
stats := source.Export()
|
||||
|
||||
sort.SliceStable(stats.List, func(p, q int) bool {
|
||||
return strings.ToLower(stats.List[p]) < strings.ToLower(stats.List[q])
|
||||
})
|
||||
|
||||
var startIndex, stopIndex int
|
||||
|
||||
if page == -1 {
|
||||
startIndex = 0
|
||||
stopIndex = len(stats.List)
|
||||
} else {
|
||||
startIndex = ((page - 1) * int(PageLength))
|
||||
stopIndex = (startIndex + int(PageLength))
|
||||
}
|
||||
|
||||
if startIndex > len(stats.List)-1 {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
|
||||
if stopIndex > len(stats.List) {
|
||||
stopIndex = len(stats.List)
|
||||
}
|
||||
|
||||
a := make([]timesServed, (stopIndex - startIndex))
|
||||
|
||||
for k, v := range stats.List[startIndex:stopIndex] {
|
||||
a[k] = timesServed{v, stats.Count[v], stats.Size[v], stats.Times[v]}
|
||||
}
|
||||
|
||||
r, err := json.MarshalIndent(a, "", " ")
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func serveStatsPage(args []string, stats *serveStats) httprouter.Handle {
|
||||
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||
startTime := time.Now()
|
||||
|
||||
page, err := strconv.Atoi(p.ByName("page"))
|
||||
if err != nil || page == 0 {
|
||||
page = -1
|
||||
}
|
||||
|
||||
response, err := stats.listFiles(page)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
|
||||
serverError(w, r, nil)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
w.Write(response)
|
||||
|
||||
if Verbose {
|
||||
fmt.Printf("%s | Served statistics page (%s) to %s in %s\n",
|
||||
startTime.Format(logDate),
|
||||
humanReadableSize(len(response)),
|
||||
realIP(r),
|
||||
time.Since(startTime).Round(time.Microsecond),
|
||||
)
|
||||
}
|
||||
|
||||
if StatisticsFile != "" {
|
||||
stats.exportFile(StatisticsFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func registerStatsHandlers(mux *httprouter.Router, args []string, stats *serveStats) {
|
||||
mux.GET("/stats", serveStatsPage(args, stats))
|
||||
if PageLength != 0 {
|
||||
mux.GET("/stats/:page", serveStatsPage(args, stats))
|
||||
}
|
||||
}
|
||||
46
cmd/web.go
46
cmd/web.go
|
|
@ -12,14 +12,12 @@ import (
|
|||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
|
|
@ -40,7 +38,7 @@ const (
|
|||
timeout time.Duration = 10 * time.Second
|
||||
)
|
||||
|
||||
func serveStaticFile(paths []string, stats *serveStats, cache *fileCache) httprouter.Handle {
|
||||
func serveStaticFile(paths []string, cache *fileCache) httprouter.Handle {
|
||||
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||
path := strings.TrimPrefix(r.URL.Path, sourcePrefix)
|
||||
|
||||
|
|
@ -122,11 +120,6 @@ func serveStaticFile(paths []string, stats *serveStats, cache *fileCache) httpro
|
|||
time.Since(startTime).Round(time.Microsecond),
|
||||
)
|
||||
}
|
||||
|
||||
if Statistics {
|
||||
stats.incrementCounter(filePath, startTime, fileSize)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -368,14 +361,6 @@ func ServePage(args []string) error {
|
|||
WriteTimeout: 5 * time.Minute,
|
||||
}
|
||||
|
||||
stats := &serveStats{
|
||||
mutex: sync.RWMutex{},
|
||||
list: []string{},
|
||||
count: make(map[string]uint32),
|
||||
size: make(map[string]string),
|
||||
times: make(map[string][]string),
|
||||
}
|
||||
|
||||
mux.PanicHandler = serverErrorHandler()
|
||||
|
||||
mux.GET("/", serveRoot(paths, regexes, cache, formats))
|
||||
|
|
@ -386,7 +371,7 @@ func ServePage(args []string) error {
|
|||
|
||||
mux.GET(mediaPrefix+"/*media", serveMedia(paths, regexes, formats))
|
||||
|
||||
mux.GET(sourcePrefix+"/*static", serveStaticFile(paths, stats, cache))
|
||||
mux.GET(sourcePrefix+"/*static", serveStaticFile(paths, cache))
|
||||
|
||||
mux.GET("/version", serveVersion())
|
||||
|
||||
|
|
@ -406,33 +391,6 @@ func ServePage(args []string) error {
|
|||
fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n")
|
||||
}
|
||||
|
||||
if Statistics {
|
||||
if StatisticsFile != "" {
|
||||
StatisticsFile, err = normalizePath(StatisticsFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err := stats.importFile(StatisticsFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gracefulShutdown := make(chan os.Signal, 1)
|
||||
signal.Notify(gracefulShutdown, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
<-gracefulShutdown
|
||||
|
||||
stats.exportFile(StatisticsFile)
|
||||
|
||||
os.Exit(0)
|
||||
}()
|
||||
}
|
||||
|
||||
registerStatsHandlers(mux, args, stats)
|
||||
}
|
||||
|
||||
err = srv.ListenAndServe()
|
||||
if !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
|
|
|
|||
Loading…
Reference in New Issue