Rename cache to index for more accurate terminology

This commit is contained in:
Seednode 2023-09-28 10:09:45 -05:00
parent 19c2e616fc
commit 1d0e4634c7
7 changed files with 267 additions and 273 deletions

View File

@ -1,5 +1,4 @@
## About
Sometimes, you just need a way to randomly display media from your filesystem.
Simply point this tool at one or more directories, and then open the specified port (default `8080`) in your browser.
@ -18,18 +17,7 @@ x86_64 and ARM Docker images of latest version: `oci.seedno.de/seednode/roulette
Dockerfile available [here](https://git.seedno.de/seednode/roulette/raw/branch/master/docker/Dockerfile).
## Caching
If the `-c|--cache` flag is passed, the indices of all specified paths will be cached on start.
This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
The cache can be regenerated at any time by accessing the `/clear_cache` endpoint.
If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated.
## Filtering
You can provide a comma-delimited string of alphanumeric patterns to match via the `include=` query parameter, assuming the `-f|--filter` flag is enabled.
Only filenames matching one or more of the patterns will be served.
@ -42,8 +30,16 @@ You can combine these two parameters, with exclusions taking priority over inclu
Both filtering parameters ignore the file extension and full path; they only compare against the bare filename.
## Info
## Indexing
If the `-i|--indexing` flag is passed, all specified paths will be indexed on start.
This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
The index can be regenerated at any time by accessing the `/rebuild_index` endpoint.
If `--index-file` is set, the index will be loaded from the specified file on start, and written to the file whenever it is re-generated.
## Info
If the `-i|--info` flag is passed, six additional endpoints are registered.
The first of these—`/html` and `/json`—return the contents of the index, in HTML and JSON formats respectively.
@ -55,7 +51,6 @@ This can prove useful when confirming whether the index is generated successfull
The remaining four endpoints—`/available_extensions`, `/enabled_extensions`, `/available_mime_types` and `/enabled_mime_types`—return information about the registered file types.
## Refresh
If the `--refresh` flag is passed and a positive-value `refresh=<integer><unit>` query parameter is provided, the page will reload after that interval.
This can be used to generate a sort of slideshow of files.
@ -76,7 +71,6 @@ That said, this has not been tested to any real extent, so only pass this flag o
Enjoy!
## Sorting
You can specify a sorting direction via the `sort=` query parameter, assuming the `-s|--sort` flag is enabled.
A value of `sort=asc` means files will be served in ascending order (lowest-numbered to highest).
@ -115,8 +109,6 @@ Flags:
-a, --all enable all supported file types
--audio enable support for audio files
-b, --bind string address to bind to (default "0.0.0.0")
-c, --cache generate directory cache at startup
--cache-file string path to optional persistent cache file
--case-sensitive use case-sensitive matching for filters
--code enable support for source code files
--code-theme string theme for source code syntax highlighting (default "solarized-dark256")
@ -126,6 +118,8 @@ Flags:
--handlers display registered handlers (for debugging)
-h, --help help for roulette
--images enable support for image files
-c, --index generate index of supported file paths at startup
--index-file string path to optional persistent index file
-i, --info expose informational endpoints
--max-directory-scans int number of directories to scan at once (default 32)
--max-file-count int skip directories with file counts above this value (default 2147483647)

View File

@ -1,202 +0,0 @@
/*
Copyright © 2023 Seednode <seednode@seedno.de>
*/
package cmd
import (
"encoding/gob"
"fmt"
"net/http"
"os"
"sync"
"time"
"github.com/julienschmidt/httprouter"
"github.com/klauspost/compress/zstd"
"seedno.de/seednode/roulette/types"
)
type fileCache struct {
mutex sync.RWMutex
list []string
}
func (cache *fileCache) List() []string {
cache.mutex.RLock()
val := make([]string, len(cache.list))
copy(val, cache.list)
cache.mutex.RUnlock()
return val
}
func (cache *fileCache) remove(path string) {
cache.mutex.RLock()
tempIndex := make([]string, len(cache.list))
copy(tempIndex, cache.list)
cache.mutex.RUnlock()
var position int
for k, v := range tempIndex {
if path == v {
position = k
break
}
}
tempIndex[position] = tempIndex[len(tempIndex)-1]
cache.mutex.Lock()
cache.list = make([]string, len(tempIndex)-1)
copy(cache.list, tempIndex[:len(tempIndex)-1])
cache.mutex.Unlock()
}
func (cache *fileCache) set(val []string) {
length := len(val)
if length < 1 {
return
}
cache.mutex.Lock()
cache.list = make([]string, length)
copy(cache.list, val)
cache.mutex.Unlock()
if Cache && CacheFile != "" {
cache.Export(CacheFile)
}
}
func (cache *fileCache) clear() {
cache.mutex.Lock()
cache.list = nil
cache.mutex.Unlock()
}
func (cache *fileCache) isEmpty() bool {
cache.mutex.RLock()
length := len(cache.list)
cache.mutex.RUnlock()
return length == 0
}
func (cache *fileCache) Export(path string) error {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer file.Close()
z, err := zstd.NewWriter(file)
if err != nil {
return err
}
defer z.Close()
enc := gob.NewEncoder(z)
cache.mutex.RLock()
enc.Encode(&cache.list)
length := len(cache.list)
cache.mutex.RUnlock()
if Verbose {
fmt.Printf("%s | CACHE: Exported %d entries to %s in %s\n",
time.Now().Format(logDate),
length,
path,
time.Since(startTime),
)
}
return nil
}
func (cache *fileCache) Import(path string) error {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
return err
}
defer file.Close()
z, err := zstd.NewReader(file)
if err != nil {
return err
}
defer z.Close()
dec := gob.NewDecoder(z)
cache.mutex.Lock()
err = dec.Decode(&cache.list)
length := len(cache.list)
cache.mutex.Unlock()
if err != nil {
return err
}
if Verbose {
fmt.Printf("%s | CACHE: Imported %d entries from %s in %s\n",
time.Now().Format(logDate),
length,
path,
time.Since(startTime),
)
}
return nil
}
func serveCacheClear(args []string, cache *fileCache, formats *types.Types, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
cache.clear()
_, err := fileList(args, &filters{}, "", cache, formats)
if err != nil {
errorChannel <- err
return
}
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("Ok"))
}
}
func registerCacheHandlers(mux *httprouter.Router, args []string, cache *fileCache, formats *types.Types, errorChannel chan<- error) error {
registerHandler(mux, Prefix+"/clear_cache", serveCacheClear(args, cache, formats, errorChannel))
return nil
}
func importCache(args []string, cache *fileCache, formats *types.Types) error {
skipIndex := false
if CacheFile != "" {
err := cache.Import(CacheFile)
if err == nil {
skipIndex = true
}
}
if !skipIndex {
_, err := fileList(args, &filters{}, "", cache, formats)
if err != nil {
return err
}
}
return nil
}

View File

@ -58,14 +58,14 @@ func humanReadableSize(bytes int) string {
float64(bytes)/float64(div), "KMGTPE"[exp])
}
func kill(path string, cache *fileCache) error {
func kill(path string, index *fileIndex) error {
err := os.Remove(path)
if err != nil {
return err
}
if Cache {
cache.remove(path)
if Index {
index.remove(path)
}
return nil
@ -321,7 +321,7 @@ Poll:
return nil
}
func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Types) ([]string, error) {
func scanPaths(paths []string, sort string, index *fileIndex, formats *types.Types) ([]string, error) {
var list []string
fileChannel := make(chan string)
@ -411,37 +411,37 @@ Poll:
return list, nil
}
func fileList(paths []string, filters *filters, sort string, cache *fileCache, formats *types.Types) ([]string, error) {
func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats *types.Types) ([]string, error) {
switch {
case Cache && !cache.isEmpty() && filters.isEmpty():
return cache.List(), nil
case Cache && !cache.isEmpty() && !filters.isEmpty():
return filters.apply(cache.List()), nil
case Cache && cache.isEmpty() && !filters.isEmpty():
list, err := scanPaths(paths, sort, cache, formats)
case Index && !index.isEmpty() && filters.isEmpty():
return index.List(), nil
case Index && !index.isEmpty() && !filters.isEmpty():
return filters.apply(index.List()), nil
case Index && index.isEmpty() && !filters.isEmpty():
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}
cache.set(list)
index.set(list)
return filters.apply(cache.List()), nil
case Cache && cache.isEmpty() && filters.isEmpty():
list, err := scanPaths(paths, sort, cache, formats)
return filters.apply(index.List()), nil
case Index && index.isEmpty() && filters.isEmpty():
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}
cache.set(list)
index.set(list)
return cache.List(), nil
case !Cache && !filters.isEmpty():
list, err := scanPaths(paths, sort, cache, formats)
return index.List(), nil
case !Index && !filters.isEmpty():
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}
return filters.apply(list), nil
default:
list, err := scanPaths(paths, sort, cache, formats)
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}

202
cmd/index.go Normal file
View File

@ -0,0 +1,202 @@
/*
Copyright © 2023 Seednode <seednode@seedno.de>
*/
package cmd
import (
"encoding/gob"
"fmt"
"net/http"
"os"
"sync"
"time"
"github.com/julienschmidt/httprouter"
"github.com/klauspost/compress/zstd"
"seedno.de/seednode/roulette/types"
)
type fileIndex struct {
mutex *sync.RWMutex
list []string
}
func (index *fileIndex) List() []string {
index.mutex.RLock()
val := make([]string, len(index.list))
copy(val, index.list)
index.mutex.RUnlock()
return val
}
func (index *fileIndex) remove(path string) {
index.mutex.RLock()
tempIndex := make([]string, len(index.list))
copy(tempIndex, index.list)
index.mutex.RUnlock()
var position int
for k, v := range tempIndex {
if path == v {
position = k
break
}
}
tempIndex[position] = tempIndex[len(tempIndex)-1]
index.mutex.Lock()
index.list = make([]string, len(tempIndex)-1)
copy(index.list, tempIndex[:len(tempIndex)-1])
index.mutex.Unlock()
}
func (index *fileIndex) set(val []string) {
length := len(val)
if length < 1 {
return
}
index.mutex.Lock()
index.list = make([]string, length)
copy(index.list, val)
index.mutex.Unlock()
if Index && IndexFile != "" {
index.Export(IndexFile)
}
}
func (index *fileIndex) clear() {
index.mutex.Lock()
index.list = nil
index.mutex.Unlock()
}
func (index *fileIndex) isEmpty() bool {
index.mutex.RLock()
length := len(index.list)
index.mutex.RUnlock()
return length == 0
}
func (index *fileIndex) Export(path string) error {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer file.Close()
z, err := zstd.NewWriter(file)
if err != nil {
return err
}
defer z.Close()
enc := gob.NewEncoder(z)
index.mutex.RLock()
enc.Encode(&index.list)
length := len(index.list)
index.mutex.RUnlock()
if Verbose {
fmt.Printf("%s | INDEX: Exported %d entries to %s in %s\n",
time.Now().Format(logDate),
length,
path,
time.Since(startTime),
)
}
return nil
}
func (index *fileIndex) Import(path string) error {
startTime := time.Now()
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
return err
}
defer file.Close()
z, err := zstd.NewReader(file)
if err != nil {
return err
}
defer z.Close()
dec := gob.NewDecoder(z)
index.mutex.Lock()
err = dec.Decode(&index.list)
length := len(index.list)
index.mutex.Unlock()
if err != nil {
return err
}
if Verbose {
fmt.Printf("%s | INDEX: Imported %d entries from %s in %s\n",
time.Now().Format(logDate),
length,
path,
time.Since(startTime),
)
}
return nil
}
func serveIndexClear(args []string, index *fileIndex, formats *types.Types, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
index.clear()
_, err := fileList(args, &filters{}, "", index, formats)
if err != nil {
errorChannel <- err
return
}
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("Ok"))
}
}
func registerIndexHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats *types.Types, errorChannel chan<- error) error {
registerHandler(mux, Prefix+"/clear_index", serveIndexClear(args, index, formats, errorChannel))
return nil
}
func importIndex(args []string, index *fileIndex, formats *types.Types) error {
skipIndex := false
if IndexFile != "" {
err := index.Import(IndexFile)
if err == nil {
skipIndex = true
}
}
if !skipIndex {
_, err := fileList(args, &filters{}, "", index, formats)
if err != nil {
return err
}
}
return nil
}

View File

@ -19,13 +19,13 @@ import (
"seedno.de/seednode/roulette/types"
)
func serveIndexHtml(args []string, cache *fileCache, paginate bool) httprouter.Handle {
func serveIndexHtml(args []string, index *fileIndex, paginate bool) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
w.Header().Set("Content-Type", "text/html")
startTime := time.Now()
indexDump := cache.List()
indexDump := index.List()
fileCount := len(indexDump)
@ -133,18 +133,18 @@ func serveIndexHtml(args []string, cache *fileCache, paginate bool) httprouter.H
}
}
func serveIndexJson(args []string, index *fileCache, errorChannel chan<- error) httprouter.Handle {
func serveIndexJson(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
w.Header().Set("Content-Type", "application/json")
startTime := time.Now()
cachedFiles := index.List()
indexedFiles := index.List()
fileCount := len(cachedFiles)
fileCount := len(indexedFiles)
sort.SliceStable(cachedFiles, func(p, q int) bool {
return strings.ToLower(cachedFiles[p]) < strings.ToLower(cachedFiles[q])
sort.SliceStable(indexedFiles, func(p, q int) bool {
return strings.ToLower(indexedFiles[p]) < strings.ToLower(indexedFiles[q])
})
var startIndex, stopIndex int
@ -159,14 +159,14 @@ func serveIndexJson(args []string, index *fileCache, errorChannel chan<- error)
}
if startIndex > (fileCount - 1) {
cachedFiles = []string{}
indexedFiles = []string{}
}
if stopIndex > fileCount {
stopIndex = fileCount
}
response, err := json.MarshalIndent(cachedFiles[startIndex:stopIndex], "", " ")
response, err := json.MarshalIndent(indexedFiles[startIndex:stopIndex], "", " ")
if err != nil {
errorChannel <- err
@ -272,16 +272,16 @@ func serveEnabledMimeTypes(formats *types.Types) httprouter.Handle {
}
}
func registerInfoHandlers(mux *httprouter.Router, args []string, cache *fileCache, formats *types.Types, errorChannel chan<- error) {
if Cache {
registerHandler(mux, Prefix+"/html", serveIndexHtml(args, cache, false))
func registerInfoHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats *types.Types, errorChannel chan<- error) {
if Index {
registerHandler(mux, Prefix+"/html", serveIndexHtml(args, index, false))
if PageLength != 0 {
registerHandler(mux, Prefix+"/html/:page", serveIndexHtml(args, cache, true))
registerHandler(mux, Prefix+"/html/:page", serveIndexHtml(args, index, true))
}
registerHandler(mux, Prefix+"/json", serveIndexJson(args, cache, errorChannel))
registerHandler(mux, Prefix+"/json", serveIndexJson(args, index, errorChannel))
if PageLength != 0 {
registerHandler(mux, Prefix+"/json/:page", serveIndexJson(args, cache, errorChannel))
registerHandler(mux, Prefix+"/json/:page", serveIndexJson(args, index, errorChannel))
}
}

View File

@ -19,8 +19,6 @@ var (
All bool
Audio bool
Bind string
Cache bool
CacheFile string
CaseSensitive bool
Code bool
CodeTheme string
@ -29,6 +27,8 @@ var (
Flash bool
Handlers bool
Images bool
Index bool
IndexFile string
Info bool
MaxDirScans int
MaxFileScans int
@ -87,8 +87,6 @@ func init() {
rootCmd.Flags().BoolVarP(&All, "all", "a", false, "enable all supported file types")
rootCmd.Flags().BoolVar(&Audio, "audio", false, "enable support for audio files")
rootCmd.Flags().StringVarP(&Bind, "bind", "b", "0.0.0.0", "address to bind to")
rootCmd.Flags().BoolVarP(&Cache, "cache", "c", false, "generate directory cache at startup")
rootCmd.Flags().StringVar(&CacheFile, "cache-file", "", "path to optional persistent cache file")
rootCmd.Flags().BoolVar(&CaseSensitive, "case-sensitive", false, "use case-sensitive matching for filters")
rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
@ -97,6 +95,8 @@ func init() {
rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)")
rootCmd.Flags().BoolVar(&Handlers, "handlers", false, "display registered handlers (for debugging)")
rootCmd.Flags().BoolVar(&Images, "images", false, "enable support for image files")
rootCmd.Flags().BoolVarP(&Index, "index", "c", false, "generate index of supported file paths at startup")
rootCmd.Flags().StringVar(&IndexFile, "index-file", "", "path to optional persistent index file")
rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints")
rootCmd.Flags().IntVar(&MaxDirScans, "max-directory-scans", 32, "number of directories to scan at once")
rootCmd.Flags().IntVar(&MaxFileScans, "max-file-scans", 256, "number of files to scan at once")

View File

@ -49,7 +49,7 @@ func preparePath(path string) string {
return mediaPrefix + path
}
func serveStaticFile(paths []string, cache *fileCache, errorChannel chan<- error) httprouter.Handle {
func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
prefix := Prefix + sourcePrefix
@ -129,7 +129,7 @@ func serveStaticFile(paths []string, cache *fileCache, errorChannel chan<- error
}
if Russian && refererUri != "" {
err = kill(filePath, cache)
err = kill(filePath, index)
if err != nil {
errorChannel <- err
@ -152,7 +152,7 @@ func serveStaticFile(paths []string, cache *fileCache, errorChannel chan<- error
}
}
func serveRoot(paths []string, regexes *regexes, cache *fileCache, formats *types.Types, errorChannel chan<- error) httprouter.Handle {
func serveRoot(paths []string, regexes *regexes, index *fileIndex, formats *types.Types, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
refererUri, err := stripQueryParams(refererToUri(r.Referer()))
if err != nil {
@ -187,7 +187,7 @@ func serveRoot(paths []string, regexes *regexes, cache *fileCache, formats *type
}
}
list, err := fileList(paths, filters, sortOrder, cache, formats)
list, err := fileList(paths, filters, sortOrder, index, formats)
if err != nil {
errorChannel <- err
@ -235,7 +235,7 @@ func serveRoot(paths []string, regexes *regexes, cache *fileCache, formats *type
}
}
func serveMedia(paths []string, regexes *regexes, cache *fileCache, formats *types.Types, errorChannel chan<- error) httprouter.Handle {
func serveMedia(paths []string, regexes *regexes, index *fileIndex, formats *types.Types, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
filters := &filters{
included: splitQueryParams(r.URL.Query().Get("include"), regexes),
@ -346,7 +346,7 @@ func serveMedia(paths []string, regexes *regexes, cache *fileCache, formats *typ
}
if Russian {
kill(path, cache)
kill(path, index)
}
}
}
@ -461,8 +461,8 @@ func ServePage(args []string) error {
listenHost := net.JoinHostPort(Bind, strconv.Itoa(Port))
cache := &fileCache{
mutex: sync.RWMutex{},
index := &fileIndex{
mutex: &sync.RWMutex{},
list: []string{},
}
@ -480,7 +480,7 @@ func ServePage(args []string) error {
errorChannel := make(chan error)
registerHandler(mux, Prefix, serveRoot(paths, regexes, cache, formats, errorChannel))
registerHandler(mux, Prefix, serveRoot(paths, regexes, index, formats, errorChannel))
Prefix = strings.TrimSuffix(Prefix, "/")
@ -492,21 +492,21 @@ func ServePage(args []string) error {
registerHandler(mux, Prefix+"/favicon.ico", serveFavicons())
registerHandler(mux, Prefix+mediaPrefix+"/*media", serveMedia(paths, regexes, cache, formats, errorChannel))
registerHandler(mux, Prefix+mediaPrefix+"/*media", serveMedia(paths, regexes, index, formats, errorChannel))
registerHandler(mux, Prefix+sourcePrefix+"/*static", serveStaticFile(paths, cache, errorChannel))
registerHandler(mux, Prefix+sourcePrefix+"/*static", serveStaticFile(paths, index, errorChannel))
registerHandler(mux, Prefix+"/version", serveVersion())
if Cache {
err = registerCacheHandlers(mux, args, cache, formats, errorChannel)
if Index {
err = registerIndexHandlers(mux, args, index, formats, errorChannel)
if err != nil {
return err
}
}
if Info {
registerInfoHandlers(mux, args, cache, formats, errorChannel)
registerInfoHandlers(mux, args, index, formats, errorChannel)
}
if Profile {
@ -517,7 +517,7 @@ func ServePage(args []string) error {
fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n")
}
err = importCache(paths, cache, formats)
err = importIndex(paths, index, formats)
if err != nil {
return err
}