Compare commits

..

2 Commits

Author SHA1 Message Date
Seednode ba827430bb Change --theme to --code-theme 2023-09-15 01:31:23 -05:00
Seednode 90a480fad4 Simplify caching, allow use of filters with cache 2023-09-15 01:06:52 -05:00
7 changed files with 80 additions and 129 deletions

View File

@ -24,8 +24,6 @@ If the `-c|--cache` flag is passed, the indices of all specified paths will be c
This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests. This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request.
The cache can be regenerated at any time by accessing the `/clear_cache` endpoint. The cache can be regenerated at any time by accessing the `/clear_cache` endpoint.
If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated. If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated.
@ -120,6 +118,7 @@ Flags:
-c, --cache generate directory cache at startup -c, --cache generate directory cache at startup
--cache-file string path to optional persistent cache file --cache-file string path to optional persistent cache file
--code enable support for source code files --code enable support for source code files
--code-theme string theme for source code syntax highlighting (default "solarized-dark256")
-f, --filter enable filtering -f, --filter enable filtering
--flash enable support for shockwave flash files (via ruffle.rs) --flash enable support for shockwave flash files (via ruffle.rs)
--handlers display registered handlers (for debugging) --handlers display registered handlers (for debugging)
@ -137,7 +136,6 @@ Flags:
--russian remove selected images after serving --russian remove selected images after serving
-s, --sort enable sorting -s, --sort enable sorting
--text enable support for text files --text enable support for text files
--theme string theme for source code syntax highlighting (default "solarized-dark256")
-v, --verbose log accessed files and other information to stdout -v, --verbose log accessed files and other information to stdout
-V, --version display version and exit -V, --version display version and exit
--video enable support for video files --video enable support for video files

View File

@ -22,7 +22,8 @@ type fileCache struct {
func (cache *fileCache) List() []string { func (cache *fileCache) List() []string {
cache.mutex.RLock() cache.mutex.RLock()
val := cache.list val := make([]string, len(cache.list))
copy(val, cache.list)
cache.mutex.RUnlock() cache.mutex.RUnlock()
return val return val
@ -53,8 +54,15 @@ func (cache *fileCache) remove(path string) {
} }
func (cache *fileCache) set(val []string) { func (cache *fileCache) set(val []string) {
length := len(val)
if length < 1 {
return
}
cache.mutex.Lock() cache.mutex.Lock()
cache.list = val cache.list = make([]string, length)
copy(cache.list, val)
cache.mutex.Unlock() cache.mutex.Unlock()
} }

View File

@ -43,12 +43,12 @@ type concurrency struct {
type files struct { type files struct {
mutex sync.RWMutex mutex sync.RWMutex
list map[string][]string list []string
} }
func (f *files) append(directory, path string) { func (f *files) append(path string) {
f.mutex.Lock() f.mutex.Lock()
f.list[directory] = append(f.list[directory], path) f.list = append(f.list, path)
f.mutex.Unlock() f.mutex.Unlock()
} }
@ -99,63 +99,6 @@ func preparePath(path string) string {
return mediaPrefix + path return mediaPrefix + path
} }
func appendPath(directory, path string, files *files, stats *scanStats, formats *types.Types) {
if !formats.Validate(path) {
stats.filesSkipped.Add(1)
return
}
files.append(directory, path)
stats.filesMatched.Add(1)
}
func appendPaths(path string, files *files, filters *filters, stats *scanStats, formats *types.Types) error {
absolutePath, err := filepath.Abs(path)
if err != nil {
return err
}
directory, filename := filepath.Split(absolutePath)
filename = strings.ToLower(filename)
if filters.hasExcludes() {
for i := 0; i < len(filters.excluded); i++ {
if strings.Contains(
filename,
filters.excluded[i],
) {
stats.filesSkipped.Add(1)
return nil
}
}
}
if filters.hasIncludes() {
for i := 0; i < len(filters.included); i++ {
if strings.Contains(
filename,
filters.included[i],
) {
appendPath(directory, path, files, stats, formats)
return nil
}
}
stats.filesSkipped.Add(1)
return nil
}
appendPath(directory, path, files, stats, formats)
return nil
}
func newFile(paths []string, filters *filters, sortOrder string, regexes *regexes, cache *fileCache, formats *types.Types) (string, error) { func newFile(paths []string, filters *filters, sortOrder string, regexes *regexes, cache *fileCache, formats *types.Types) (string, error) {
path, err := pickFile(paths, filters, sortOrder, cache, formats) path, err := pickFile(paths, filters, sortOrder, cache, formats)
if err != nil { if err != nil {
@ -351,7 +294,7 @@ func pathCount(path string) (uint32, uint32, error) {
return files, directories, nil return files, directories, nil
} }
func scanPath(path string, files *files, filters *filters, stats *scanStats, concurrency *concurrency, formats *types.Types) error { func scanPath(path string, files *files, stats *scanStats, concurrency *concurrency, formats *types.Types) error {
var wg sync.WaitGroup var wg sync.WaitGroup
err := filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error { err := filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error {
@ -378,10 +321,15 @@ func scanPath(path string, files *files, filters *filters, stats *scanStats, con
fmt.Println(err) fmt.Println(err)
} }
err = appendPaths(path, files, filters, stats, formats) if !formats.Validate(path) {
if err != nil { stats.filesSkipped.Add(1)
fmt.Println(err)
return
} }
files.append(path)
stats.filesMatched.Add(1)
}() }()
case info.IsDir(): case info.IsDir():
files, directories, err := pathCount(p) files, directories, err := pathCount(p)
@ -412,16 +360,10 @@ func scanPath(path string, files *files, filters *filters, stats *scanStats, con
return nil return nil
} }
func fileList(paths []string, filters *filters, sort string, cache *fileCache, formats *types.Types) ([]string, bool) { func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Types) []string {
if Cache && filters.isEmpty() && !cache.isEmpty() {
return cache.List(), true
}
var fileList []string
files := &files{ files := &files{
mutex: sync.RWMutex{}, mutex: sync.RWMutex{},
list: make(map[string][]string), list: []string{},
} }
stats := &scanStats{ stats := &scanStats{
@ -451,7 +393,7 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f
wg.Done() wg.Done()
}() }()
err := scanPath(paths[i], files, filters, stats, concurrency, formats) err := scanPath(paths[i], files, stats, concurrency, formats)
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
} }
@ -460,10 +402,9 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f
wg.Wait() wg.Wait()
fileList = prepareDirectories(files, sort)
if stats.filesMatched.Load() < 1 { if stats.filesMatched.Load() < 1 {
return []string{}, false fmt.Println("No files matched")
return []string{}
} }
if Verbose { if Verbose {
@ -477,35 +418,32 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f
) )
} }
if Cache && filters.isEmpty() { return files.list
cache.set(fileList)
} }
return fileList, false func fileList(paths []string, filters *filters, sort string, cache *fileCache, formats *types.Types) []string {
switch {
case Cache && !cache.isEmpty() && filters.isEmpty():
return cache.List()
case Cache && !cache.isEmpty() && !filters.isEmpty():
return filters.apply(cache.List())
case Cache && cache.isEmpty() && !filters.isEmpty():
cache.set(scanPaths(paths, sort, cache, formats))
return filters.apply(cache.List())
case Cache && cache.isEmpty() && filters.isEmpty():
cache.set(scanPaths(paths, sort, cache, formats))
return cache.List()
case !Cache && !filters.isEmpty():
return filters.apply(scanPaths(paths, sort, cache, formats))
default:
return scanPaths(paths, sort, cache, formats)
} }
func prepareDirectories(files *files, sort string) []string {
directories := []string{}
keys := make([]string, len(files.list))
i := 0
for k := range files.list {
keys[i] = k
i++
}
for i := 0; i < len(keys); i++ {
directories = append(directories, files.list[keys[i]]...)
}
return directories
} }
func pickFile(args []string, filters *filters, sort string, cache *fileCache, formats *types.Types) (string, error) { func pickFile(args []string, filters *filters, sort string, cache *fileCache, formats *types.Types) (string, error) {
fileList, fromCache := fileList(args, filters, sort, cache, formats) list := fileList(args, filters, sort, cache, formats)
fileCount := len(fileList) fileCount := len(list)
if fileCount < 1 { if fileCount < 1 {
return "", ErrNoMediaFound return "", ErrNoMediaFound
@ -521,27 +459,7 @@ func pickFile(args []string, filters *filters, sort string, cache *fileCache, fo
return "", err return "", err
} }
for i := 0; i < fileCount; i++ { return list[val], nil
switch {
case val >= fileCount:
val = 0
case val < fileCount-1:
val++
}
path := fileList[val]
switch {
case !fromCache && formats.Validate(path):
return path, nil
case !fromCache:
continue
default:
return path, nil
}
}
return "", ErrNoMediaFound
} }
func normalizePath(path string) (string, error) { func normalizePath(path string) (string, error) {

View File

@ -4,7 +4,10 @@ Copyright © 2023 Seednode <seednode@seedno.de>
package cmd package cmd
import "strings" import (
"slices"
"strings"
)
type filters struct { type filters struct {
included []string included []string
@ -30,3 +33,27 @@ func (filters *filters) hasExcludes() bool {
func (filters *filters) excludes() string { func (filters *filters) excludes() string {
return strings.Join(filters.excluded, ",") return strings.Join(filters.excluded, ",")
} }
func (filters *filters) apply(fileList []string) []string {
result := make([]string, len(fileList))
copy(result, fileList)
if filters.hasExcludes() {
for _, exclude := range filters.excluded {
result = slices.DeleteFunc(fileList, func(s string) bool {
return strings.Contains(strings.ToLower(s), strings.ToLower(exclude))
})
}
}
if filters.hasIncludes() {
for _, include := range filters.included {
result = slices.DeleteFunc(fileList, func(s string) bool {
return !strings.Contains(strings.ToLower(s), strings.ToLower(include))
})
}
}
return result
}

View File

@ -11,7 +11,7 @@ import (
) )
const ( const (
ReleaseVersion string = "0.85.1" ReleaseVersion string = "0.87.0"
) )
var ( var (
@ -21,6 +21,7 @@ var (
Cache bool Cache bool
CacheFile string CacheFile string
Code bool Code bool
CodeTheme string
Filtering bool Filtering bool
Flash bool Flash bool
Handlers bool Handlers bool
@ -37,7 +38,6 @@ var (
Russian bool Russian bool
Sorting bool Sorting bool
Text bool Text bool
Theme string
Verbose bool Verbose bool
Version bool Version bool
Videos bool Videos bool
@ -71,6 +71,7 @@ func init() {
rootCmd.Flags().BoolVarP(&Cache, "cache", "c", false, "generate directory cache at startup") rootCmd.Flags().BoolVarP(&Cache, "cache", "c", false, "generate directory cache at startup")
rootCmd.Flags().StringVar(&CacheFile, "cache-file", "", "path to optional persistent cache file") rootCmd.Flags().StringVar(&CacheFile, "cache-file", "", "path to optional persistent cache file")
rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files") rootCmd.Flags().BoolVar(&Code, "code", false, "enable support for source code files")
rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
rootCmd.Flags().BoolVarP(&Filtering, "filter", "f", false, "enable filtering") rootCmd.Flags().BoolVarP(&Filtering, "filter", "f", false, "enable filtering")
rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)") rootCmd.Flags().BoolVar(&Flash, "flash", false, "enable support for shockwave flash files (via ruffle.rs)")
rootCmd.Flags().BoolVar(&Handlers, "handlers", false, "display registered handlers (for debugging)") rootCmd.Flags().BoolVar(&Handlers, "handlers", false, "display registered handlers (for debugging)")
@ -87,7 +88,6 @@ func init() {
rootCmd.Flags().BoolVar(&Russian, "russian", false, "remove selected images after serving") rootCmd.Flags().BoolVar(&Russian, "russian", false, "remove selected images after serving")
rootCmd.Flags().BoolVarP(&Sorting, "sort", "s", false, "enable sorting") rootCmd.Flags().BoolVarP(&Sorting, "sort", "s", false, "enable sorting")
rootCmd.Flags().BoolVar(&Text, "text", false, "enable support for text files") rootCmd.Flags().BoolVar(&Text, "text", false, "enable support for text files")
rootCmd.Flags().StringVar(&Theme, "theme", "solarized-dark256", "theme for source code syntax highlighting")
rootCmd.Flags().BoolVarP(&Verbose, "verbose", "v", false, "log accessed files and other information to stdout") rootCmd.Flags().BoolVarP(&Verbose, "verbose", "v", false, "log accessed files and other information to stdout")
rootCmd.Flags().BoolVarP(&Version, "version", "V", false, "display version and exit") rootCmd.Flags().BoolVarP(&Version, "version", "V", false, "display version and exit")
rootCmd.Flags().BoolVar(&Videos, "video", false, "enable support for video files") rootCmd.Flags().BoolVar(&Videos, "video", false, "enable support for video files")

View File

@ -331,7 +331,7 @@ func ServePage(args []string) error {
} }
if Code || All { if Code || All {
formats.Add(code.New(Theme)) formats.Add(code.New(CodeTheme))
} }
if Flash || All { if Flash || All {

Binary file not shown.