Simplify caching, allow use of filters with cache

README.md

@@ -24,8 +24,6 @@ If the `-c|--cache` flag is passed, the indices of all specified paths will be c
 
 This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests.
 
-If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request.
-
 The cache can be regenerated at any time by accessing the `/clear_cache` endpoint.
 
 If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated.

cmd/cache.go

@@ -22,7 +22,8 @@ type fileCache struct {
 
 func (cache *fileCache) List() []string {
 	cache.mutex.RLock()
-	val := cache.list
+	val := make([]string, len(cache.list))
+	copy(val, cache.list)
 	cache.mutex.RUnlock()
 
 	return val
@@ -53,8 +54,15 @@ func (cache *fileCache) remove(path string) {
 }
 
 func (cache *fileCache) set(val []string) {
+	length := len(val)
+
+	if length < 1 {
+		return
+	}
+
 	cache.mutex.Lock()
-	cache.list = val
+	cache.list = make([]string, length)
+	copy(cache.list, val)
 	cache.mutex.Unlock()
 }
 

cmd/files.go

@@ -43,12 +43,12 @@ type concurrency struct {
 
 type files struct {
 	mutex sync.RWMutex
-	list  map[string][]string
+	list  []string
 }
 
-func (f *files) append(directory, path string) {
+func (f *files) append(path string) {
 	f.mutex.Lock()
-	f.list[directory] = append(f.list[directory], path)
+	f.list = append(f.list, path)
 	f.mutex.Unlock()
 }
 
@@ -99,63 +99,6 @@ func preparePath(path string) string {
 	return mediaPrefix + path
 }
 
-func appendPath(directory, path string, files *files, stats *scanStats, formats *types.Types) {
-	if !formats.Validate(path) {
-		stats.filesSkipped.Add(1)
-
-		return
-	}
-
-	files.append(directory, path)
-
-	stats.filesMatched.Add(1)
-}
-
-func appendPaths(path string, files *files, filters *filters, stats *scanStats, formats *types.Types) error {
-	absolutePath, err := filepath.Abs(path)
-	if err != nil {
-		return err
-	}
-
-	directory, filename := filepath.Split(absolutePath)
-
-	filename = strings.ToLower(filename)
-
-	if filters.hasExcludes() {
-		for i := 0; i < len(filters.excluded); i++ {
-			if strings.Contains(
-				filename,
-				filters.excluded[i],
-			) {
-				stats.filesSkipped.Add(1)
-
-				return nil
-			}
-		}
-	}
-
-	if filters.hasIncludes() {
-		for i := 0; i < len(filters.included); i++ {
-			if strings.Contains(
-				filename,
-				filters.included[i],
-			) {
-				appendPath(directory, path, files, stats, formats)
-
-				return nil
-			}
-		}
-
-		stats.filesSkipped.Add(1)
-
-		return nil
-	}
-
-	appendPath(directory, path, files, stats, formats)
-
-	return nil
-}
-
 func newFile(paths []string, filters *filters, sortOrder string, regexes *regexes, cache *fileCache, formats *types.Types) (string, error) {
 	path, err := pickFile(paths, filters, sortOrder, cache, formats)
 	if err != nil {
@@ -351,7 +294,7 @@ func pathCount(path string) (uint32, uint32, error) {
 	return files, directories, nil
 }
 
-func scanPath(path string, files *files, filters *filters, stats *scanStats, concurrency *concurrency, formats *types.Types) error {
+func scanPath(path string, files *files, stats *scanStats, concurrency *concurrency, formats *types.Types) error {
 	var wg sync.WaitGroup
 
 	err := filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error {
@@ -378,10 +321,15 @@ func scanPath(path string, files *files, filters *filters, stats *scanStats, con
 					fmt.Println(err)
 				}
 
-				err = appendPaths(path, files, filters, stats, formats)
+				if !formats.Validate(path) {
-				if err != nil {
+					stats.filesSkipped.Add(1)
-					fmt.Println(err)
+
+					return
 				}
+
+				files.append(path)
+
+				stats.filesMatched.Add(1)
 			}()
 		case info.IsDir():
 			files, directories, err := pathCount(p)
@@ -412,16 +360,10 @@ func scanPath(path string, files *files, filters *filters, stats *scanStats, con
 	return nil
 }
 
-func fileList(paths []string, filters *filters, sort string, cache *fileCache, formats *types.Types) ([]string, bool) {
+func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Types) []string {
-	if Cache && filters.isEmpty() && !cache.isEmpty() {
-		return cache.List(), true
-	}
-
-	var fileList []string
-
 	files := &files{
 		mutex: sync.RWMutex{},
-		list:  make(map[string][]string),
+		list:  []string{},
 	}
 
 	stats := &scanStats{
@@ -451,7 +393,7 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f
 				wg.Done()
 			}()
 
-			err := scanPath(paths[i], files, filters, stats, concurrency, formats)
+			err := scanPath(paths[i], files, stats, concurrency, formats)
 			if err != nil {
 				fmt.Println(err)
 			}
@@ -460,10 +402,9 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f
 
 	wg.Wait()
 
-	fileList = prepareDirectories(files, sort)
-
 	if stats.filesMatched.Load() < 1 {
-		return []string{}, false
+		fmt.Println("No files matched")
+		return []string{}
 	}
 
 	if Verbose {
@@ -477,35 +418,32 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f
 		)
 	}
 
-	if Cache && filters.isEmpty() {
+	return files.list
-		cache.set(fileList)
-	}
-
-	return fileList, false
 }
 
-func prepareDirectories(files *files, sort string) []string {
+func fileList(paths []string, filters *filters, sort string, cache *fileCache, formats *types.Types) []string {
-	directories := []string{}
+	switch {
-
+	case Cache && !cache.isEmpty() && filters.isEmpty():
-	keys := make([]string, len(files.list))
+		return cache.List()
-
+	case Cache && !cache.isEmpty() && !filters.isEmpty():
-	i := 0
+		return filters.apply(cache.List())
-	for k := range files.list {
+	case Cache && cache.isEmpty() && !filters.isEmpty():
-		keys[i] = k
+		cache.set(scanPaths(paths, sort, cache, formats))
-		i++
+		return filters.apply(cache.List())
+	case Cache && cache.isEmpty() && filters.isEmpty():
+		cache.set(scanPaths(paths, sort, cache, formats))
+		return cache.List()
+	case !Cache && !filters.isEmpty():
+		return filters.apply(scanPaths(paths, sort, cache, formats))
+	default:
+		return scanPaths(paths, sort, cache, formats)
 	}
-
-	for i := 0; i < len(keys); i++ {
-		directories = append(directories, files.list[keys[i]]...)
-	}
-
-	return directories
 }
 
 func pickFile(args []string, filters *filters, sort string, cache *fileCache, formats *types.Types) (string, error) {
-	fileList, fromCache := fileList(args, filters, sort, cache, formats)
+	list := fileList(args, filters, sort, cache, formats)
 
-	fileCount := len(fileList)
+	fileCount := len(list)
 
 	if fileCount < 1 {
 		return "", ErrNoMediaFound
@@ -521,27 +459,7 @@ func pickFile(args []string, filters *filters, sort string, cache *fileCache, fo
 		return "", err
 	}
 
-	for i := 0; i < fileCount; i++ {
+	return list[val], nil
-		switch {
-		case val >= fileCount:
-			val = 0
-		case val < fileCount-1:
-			val++
-		}
-
-		path := fileList[val]
-
-		switch {
-		case !fromCache && formats.Validate(path):
-			return path, nil
-		case !fromCache:
-			continue
-		default:
-			return path, nil
-		}
-	}
-
-	return "", ErrNoMediaFound
 }
 
 func normalizePath(path string) (string, error) {

cmd/filters.go

@@ -4,7 +4,10 @@ Copyright © 2023 Seednode <seednode@seedno.de>
 
 package cmd
 
-import "strings"
+import (
+	"slices"
+	"strings"
+)
 
 type filters struct {
 	included []string
@@ -30,3 +33,27 @@ func (filters *filters) hasExcludes() bool {
 func (filters *filters) excludes() string {
 	return strings.Join(filters.excluded, ",")
 }
+
+func (filters *filters) apply(fileList []string) []string {
+	result := make([]string, len(fileList))
+
+	copy(result, fileList)
+
+	if filters.hasExcludes() {
+		for _, exclude := range filters.excluded {
+			result = slices.DeleteFunc(fileList, func(s string) bool {
+				return strings.Contains(strings.ToLower(s), strings.ToLower(exclude))
+			})
+		}
+	}
+
+	if filters.hasIncludes() {
+		for _, include := range filters.included {
+			result = slices.DeleteFunc(fileList, func(s string) bool {
+				return !strings.Contains(strings.ToLower(s), strings.ToLower(include))
+			})
+		}
+	}
+
+	return result
+}

cmd/root.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	ReleaseVersion string = "0.85.1"
+	ReleaseVersion string = "0.86.0"
 )
 
 var (