SeanOMik
/
roulette
mirror of https://git.seedno.de/seednode/roulette.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removed statistics (sorry) pending full rewrite

This commit is contained in:
Seednode 2023-09-13 17:02:43 -05:00
parent d2d9418b8b
commit ce99374a9f
5 changed files with 49 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
README.md
View File

@ -18,6 +18,18 @@ x86_64 and ARM Docker images of latest version: `oci.seedno.de/seednode/roulette
Dockerfile available [here](https://git.seedno.de/seednode/roulette/raw/branch/master/docker/Dockerfile). Dockerfile available [here](https://git.seedno.de/seednode/roulette/raw/branch/master/docker/Dockerfile).
## Caching
If the `-c|--cache` flag is passed, the indices of all specified paths will be cached on start.
This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request.
The cache can be regenerated at any time by accessing the `/clear_cache` endpoint.
If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated.
## Filtering ## Filtering
You can provide a comma-delimited string of alphanumeric patterns to match via the `include=` query parameter, assuming the `-f|--filter` flag is enabled. You can provide a comma-delimited string of alphanumeric patterns to match via the `include=` query parameter, assuming the `-f|--filter` flag is enabled.
@ -32,6 +44,39 @@ You can also combine these two parameters, with exclusions taking priority over
Both filtering parameters ignore the file extension and full path; they only compare against the bare filename. Both filtering parameters ignore the file extension and full path; they only compare against the bare filename.
## Info
If the `-i|--info` flag is passed, six additional endpoints are registered.
The first of these—`/html` and `/json`—return the contents of the index, in HTML and JSON formats respectively.
If `--page-length` is also set, these can be viewed in paginated form by appending `/n`, e.g. `/html/5` for the fifth page.
This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index.
The remaining four endpoints—`/available_extensions`, `/enabled_extensions`, `/available_mime_types` and `/enabled_mime_types`—return information about the registered file types.
## Refresh
If a positive-value `refresh=<integer><unit>` query parameter is provided, the page will reload after that interval.
This can be used to generate a sort of slideshow of files.
Minimum accepted value is 500ms, as anything lower seems to cause inconsistent behavior. This might be changed in a future release.
Supported units are `ns`, `us`/`µs`, `ms`, `s`, `m`, and `h`.
## Russian
If the `--russian` flag is passed, everything functions exactly as you would expect.
That is, files will be deleted after being served. This is not a joke, you *will* lose data.
This uses `os.Remove()` and checks to ensure the specified file is inside one of the paths passed to `roulette`.
That said, this has not been tested to any real extent, so only pass this flag on systems you don't care about.
Enjoy!
## Sorting ## Sorting
You can specify a sorting pattern via the `sort=` query parameter, assuming the `-s|--sort` flag is enabled. You can specify a sorting pattern via the `sort=` query parameter, assuming the `-s|--sort` flag is enabled.
@ -54,59 +99,6 @@ If any other (or no) value is provided, the selected file will be random.
Note: These patterns require sequentially-numbered files matching the following pattern: `filename###.extension`. Note: These patterns require sequentially-numbered files matching the following pattern: `filename###.extension`.
## Refresh
If a positive-value `refresh=<integer><unit>` query parameter is provided, the page will reload after that interval.
This can be used to generate a sort of slideshow of files.
Minimum accepted value is 500ms, as anything lower seems to cause inconsistent behavior. This might be changed in a future release.
Supported units are `ns`, `us`/`µs`, `ms`, `s`, `m`, and `h`.
## Caching
If the `-c|--cache` flag is passed, the indices of all specified paths will be cached on start.
This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request.
The cache can be regenerated at any time by accessing the `/clear_cache` endpoint.
If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated.
## Info
If the `-i|--info` flag is passed, six additional endpoints are registered.
The first of these—`/html` and `/json`—return the contents of the index, in HTML and JSON formats respectively.
If `--page-length` is also set, these can be viewed in paginated form by appending `/n`, e.g. `/html/5` for the fifth page.
This can prove useful when confirming whether the index is generated successfully, or whether a given file is in the index.
The remaining four endpoints—`/available_extensions`, `/enabled_extensions`, `/available_mime_types` and `/enabled_mime_types`—return information about the registered file types.
## Statistics
If the `--stats` flag is passed, an additional endpoint, `/stats`, is registered.
When accessed, this endpoint returns a JSON document listing every file served, along with the number of times it has been served, its filesize, and timestamps of when it was served.
If `--page-length` is also set, this can be viewed in paginated form by appending `/n`, e.g. `/stats/5` for the fifth page.
## Russian
If the `--russian` flag is passed, everything functions exactly as you would expect.
That is, files will be deleted after being served. This is not a joke, you *will* lose data.
This uses `os.Remove()` and checks to ensure the specified file is inside one of the paths passed to `roulette`.
That said, this has not been tested to any real extent, so only pass this flag on systems you don't care about.
Enjoy!
## Usage output ## Usage output
``` ```
Serves random media from the specified directories. Serves random media from the specified directories.
@ -134,8 +126,6 @@ Flags:
--refresh-interval string force refresh interval equal to this duration (minimum 500ms) --refresh-interval string force refresh interval equal to this duration (minimum 500ms)
--russian remove selected images after serving --russian remove selected images after serving
-s, --sort enable sorting -s, --sort enable sorting
--stats expose stats endpoint
--stats-file string path to optional persistent stats file
--text enable support for text files --text enable support for text files
-v, --verbose log accessed files and other information to stdout -v, --verbose log accessed files and other information to stdout
-V, --version display version and exit -V, --version display version and exit
@ -145,4 +135,4 @@ Flags:
## Building the Docker container ## Building the Docker container
From inside the `docker/` subdirectory, build the image using the following command: From inside the `docker/` subdirectory, build the image using the following command:
`REGISTRY=<registry url> LATEST=yes TAG=alpine ./build.sh` `REGISTRY=<registry url> LATEST=yes TAG=alpine ./build.sh`

9
cmd/files.go
View File

@ -73,15 +73,6 @@ func (splitPath *splitPath) decrement() {
splitPath.number = splitPath.number - 1 splitPath.number = splitPath.number - 1
} }
func contains(slice []string, value string) bool {
for _, v := range slice {
if v == value {
return true
}
}
return false
}
func humanReadableSize(bytes int) string { func humanReadableSize(bytes int) string {
const unit = 1000 const unit = 1000

6
cmd/root.go
View File

@ -12,7 +12,7 @@ import (
) )
const ( const (
ReleaseVersion string = "0.78.2" ReleaseVersion string = "0.79.0"
) )
var ( var (
@ -34,8 +34,6 @@ var (
RefreshInterval string RefreshInterval string
Russian bool Russian bool
Sorting bool Sorting bool
Statistics bool
StatisticsFile string
Text bool Text bool
Verbose bool Verbose bool
Version bool Version bool
@ -92,8 +90,6 @@ func init() {
rootCmd.Flags().StringVar(&RefreshInterval, "refresh-interval", "", "force refresh interval equal to this duration (minimum 500ms)") rootCmd.Flags().StringVar(&RefreshInterval, "refresh-interval", "", "force refresh interval equal to this duration (minimum 500ms)")
rootCmd.Flags().BoolVar(&Russian, "russian", false, "remove selected images after serving") rootCmd.Flags().BoolVar(&Russian, "russian", false, "remove selected images after serving")
rootCmd.Flags().BoolVarP(&Sorting, "sort", "s", false, "enable sorting") rootCmd.Flags().BoolVarP(&Sorting, "sort", "s", false, "enable sorting")
rootCmd.Flags().BoolVar(&Statistics, "stats", false, "expose stats endpoint")
rootCmd.Flags().StringVar(&StatisticsFile, "stats-file", "", "path to optional persistent stats file")
rootCmd.Flags().BoolVar(&Text, "text", false, "enable support for text files") rootCmd.Flags().BoolVar(&Text, "text", false, "enable support for text files")
rootCmd.Flags().BoolVarP(&Verbose, "verbose", "v", false, "log accessed files and other information to stdout") rootCmd.Flags().BoolVarP(&Verbose, "verbose", "v", false, "log accessed files and other information to stdout")
rootCmd.Flags().BoolVarP(&Version, "version", "V", false, "display version and exit") rootCmd.Flags().BoolVarP(&Version, "version", "V", false, "display version and exit")

253
cmd/stats.go
View File

@ -1,253 +0,0 @@
/*
Copyright © 2023 Seednode <seednode@seedno.de>
*/
package cmd
import (
"encoding/gob"
"encoding/json"
"fmt"
"net/http"
"os"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/julienschmidt/httprouter"
"github.com/klauspost/compress/zstd"
)
type serveStats struct {
mutex sync.RWMutex
list []string
count map[string]uint32
size map[string]string
times map[string][]string
}
type publicServeStats struct {
List []string
Count map[string]uint32
Size map[string]string
Times map[string][]string
}
type timesServed struct {
File string
Served uint32
Size string
Times []string
}
func (stats *serveStats) incrementCounter(file string, timestamp time.Time, filesize string) {
stats.mutex.Lock()
stats.count[file]++
stats.times[file] = append(stats.times[file], timestamp.Format(logDate))
_, exists := stats.size[file]
if !exists {
stats.size[file] = filesize
}
if !contains(stats.list, file) {
stats.list = append(stats.list, file)
}
stats.mutex.Unlock()
}
func (stats *serveStats) Import(source *publicServeStats) {
stats.mutex.Lock()
copy(stats.list, source.List)
for k, v := range source.Count {
fmt.Printf("Setting count[%s] to %d\n", k, v)
stats.count[k] = v
}
for k, v := range source.Size {
fmt.Printf("Setting size[%s] to %v\n", k, v)
stats.size[k] = v
}
for k, v := range source.Times {
fmt.Printf("Setting times[%s] to %v\n", k, v)
stats.times[k] = v
}
stats.mutex.Unlock()
}
func (source *serveStats) Export() *publicServeStats {
source.mutex.RLock()
stats := &publicServeStats{
List: make([]string, len(source.list)),
Count: make(map[string]uint32, len(source.count)),
Size: make(map[string]string, len(source.size)),
Times: make(map[string][]string, len(source.times)),
}
copy(stats.List, source.list)
for k, v := range source.count {
stats.Count[k] = v
}
for k, v := range source.size {
stats.Size[k] = v
}
for k, v := range source.times {
stats.Times[k] = v
}
source.mutex.RUnlock()
return stats
}
func (stats *serveStats) exportFile(path string) error {
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer file.Close()
z, err := zstd.NewWriter(file)
if err != nil {
return err
}
defer z.Close()
enc := gob.NewEncoder(z)
err = enc.Encode(stats.Export())
if err != nil {
return err
}
return nil
}
func (stats *serveStats) importFile(path string) error {
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
return err
}
defer file.Close()
z, err := zstd.NewReader(file)
if err != nil {
return err
}
defer z.Close()
dec := gob.NewDecoder(z)
source := &publicServeStats{
List: []string{},
Count: make(map[string]uint32),
Size: make(map[string]string),
Times: make(map[string][]string),
}
err = dec.Decode(source)
if err != nil {
return err
}
stats.Import(source)
return nil
}
func (source *serveStats) listFiles(page int) ([]byte, error) {
stats := source.Export()
sort.SliceStable(stats.List, func(p, q int) bool {
return strings.ToLower(stats.List[p]) < strings.ToLower(stats.List[q])
})
var startIndex, stopIndex int
if page == -1 {
startIndex = 0
stopIndex = len(stats.List)
} else {
startIndex = ((page - 1) * int(PageLength))
stopIndex = (startIndex + int(PageLength))
}
if startIndex > len(stats.List)-1 {
return []byte("{}"), nil
}
if stopIndex > len(stats.List) {
stopIndex = len(stats.List)
}
a := make([]timesServed, (stopIndex - startIndex))
for k, v := range stats.List[startIndex:stopIndex] {
a[k] = timesServed{v, stats.Count[v], stats.Size[v], stats.Times[v]}
}
r, err := json.MarshalIndent(a, "", " ")
if err != nil {
return []byte{}, err
}
return r, nil
}
func serveStatsPage(args []string, stats *serveStats) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
page, err := strconv.Atoi(p.ByName("page"))
if err != nil || page == 0 {
page = -1
}
response, err := stats.listFiles(page)
if err != nil {
fmt.Println(err)
serverError(w, r, nil)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(response)
if Verbose {
fmt.Printf("%s | Served statistics page (%s) to %s in %s\n",
startTime.Format(logDate),
humanReadableSize(len(response)),
realIP(r),
time.Since(startTime).Round(time.Microsecond),
)
}
if StatisticsFile != "" {
stats.exportFile(StatisticsFile)
}
}
}
func registerStatsHandlers(mux *httprouter.Router, args []string, stats *serveStats) {
mux.GET("/stats", serveStatsPage(args, stats))
if PageLength != 0 {
mux.GET("/stats/:page", serveStatsPage(args, stats))
}
}

46
cmd/web.go
View File

@ -12,14 +12,12 @@ import (
"net" "net"
"net/http" "net/http"
"os" "os"
"os/signal"
"path/filepath" "path/filepath"
"regexp" "regexp"
"runtime" "runtime"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"syscall"
"time" "time"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
@ -40,7 +38,7 @@ const (
timeout time.Duration = 10 * time.Second timeout time.Duration = 10 * time.Second
) )
func serveStaticFile(paths []string, stats *serveStats, cache *fileCache) httprouter.Handle { func serveStaticFile(paths []string, cache *fileCache) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
path := strings.TrimPrefix(r.URL.Path, sourcePrefix) path := strings.TrimPrefix(r.URL.Path, sourcePrefix)
@ -122,11 +120,6 @@ func serveStaticFile(paths []string, stats *serveStats, cache *fileCache) httpro
time.Since(startTime).Round(time.Microsecond), time.Since(startTime).Round(time.Microsecond),
) )
} }
if Statistics {
stats.incrementCounter(filePath, startTime, fileSize)
}
} }
} }
@ -368,14 +361,6 @@ func ServePage(args []string) error {
WriteTimeout: 5 * time.Minute, WriteTimeout: 5 * time.Minute,
} }
stats := &serveStats{
mutex: sync.RWMutex{},
list: []string{},
count: make(map[string]uint32),
size: make(map[string]string),
times: make(map[string][]string),
}
mux.PanicHandler = serverErrorHandler() mux.PanicHandler = serverErrorHandler()
mux.GET("/", serveRoot(paths, regexes, cache, formats)) mux.GET("/", serveRoot(paths, regexes, cache, formats))
@ -386,7 +371,7 @@ func ServePage(args []string) error {
mux.GET(mediaPrefix+"/*media", serveMedia(paths, regexes, formats)) mux.GET(mediaPrefix+"/*media", serveMedia(paths, regexes, formats))
mux.GET(sourcePrefix+"/*static", serveStaticFile(paths, stats, cache)) mux.GET(sourcePrefix+"/*static", serveStaticFile(paths, cache))
mux.GET("/version", serveVersion()) mux.GET("/version", serveVersion())
@ -406,33 +391,6 @@ func ServePage(args []string) error {
fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n") fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n")
} }
if Statistics {
if StatisticsFile != "" {
StatisticsFile, err = normalizePath(StatisticsFile)
if err != nil {
return err
}
err := stats.importFile(StatisticsFile)
if err != nil {
return err
}
gracefulShutdown := make(chan os.Signal, 1)
signal.Notify(gracefulShutdown, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-gracefulShutdown
stats.exportFile(StatisticsFile)
os.Exit(0)
}()
}
registerStatsHandlers(mux, args, stats)
}
err = srv.ListenAndServe() err = srv.ListenAndServe()
if !errors.Is(err, http.ErrServerClosed) { if !errors.Is(err, http.ErrServerClosed) {
return err return err