diff --git a/README.md b/README.md index b7a6a70..7325f27 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,6 @@ If the `-c|--cache` flag is passed, the indices of all specified paths will be c This will slightly increase the delay before the application begins responding to requests, but should significantly speed up subsequent requests. -If any `include=`/`exclude=` filters are specified in a given request, the cache will be bypassed for that specific request. - The cache can be regenerated at any time by accessing the `/clear_cache` endpoint. If `--cache-file` is set, the cache will be loaded from the specified file on start, and written to the file whenever it is re-generated. diff --git a/cmd/cache.go b/cmd/cache.go index 8cf4ab0..2e412d8 100644 --- a/cmd/cache.go +++ b/cmd/cache.go @@ -22,7 +22,8 @@ type fileCache struct { func (cache *fileCache) List() []string { cache.mutex.RLock() - val := cache.list + val := make([]string, len(cache.list)) + copy(val, cache.list) cache.mutex.RUnlock() return val @@ -53,8 +54,15 @@ func (cache *fileCache) remove(path string) { } func (cache *fileCache) set(val []string) { + length := len(val) + + if length < 1 { + return + } + cache.mutex.Lock() - cache.list = val + cache.list = make([]string, length) + copy(cache.list, val) cache.mutex.Unlock() } diff --git a/cmd/files.go b/cmd/files.go index 9b3cde8..ade8b73 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -43,12 +43,12 @@ type concurrency struct { type files struct { mutex sync.RWMutex - list map[string][]string + list []string } -func (f *files) append(directory, path string) { +func (f *files) append(path string) { f.mutex.Lock() - f.list[directory] = append(f.list[directory], path) + f.list = append(f.list, path) f.mutex.Unlock() } @@ -99,63 +99,6 @@ func preparePath(path string) string { return mediaPrefix + path } -func appendPath(directory, path string, files *files, stats *scanStats, formats *types.Types) { - if !formats.Validate(path) { - stats.filesSkipped.Add(1) - - return - } - - files.append(directory, path) - - stats.filesMatched.Add(1) -} - -func appendPaths(path string, files *files, filters *filters, stats *scanStats, formats *types.Types) error { - absolutePath, err := filepath.Abs(path) - if err != nil { - return err - } - - directory, filename := filepath.Split(absolutePath) - - filename = strings.ToLower(filename) - - if filters.hasExcludes() { - for i := 0; i < len(filters.excluded); i++ { - if strings.Contains( - filename, - filters.excluded[i], - ) { - stats.filesSkipped.Add(1) - - return nil - } - } - } - - if filters.hasIncludes() { - for i := 0; i < len(filters.included); i++ { - if strings.Contains( - filename, - filters.included[i], - ) { - appendPath(directory, path, files, stats, formats) - - return nil - } - } - - stats.filesSkipped.Add(1) - - return nil - } - - appendPath(directory, path, files, stats, formats) - - return nil -} - func newFile(paths []string, filters *filters, sortOrder string, regexes *regexes, cache *fileCache, formats *types.Types) (string, error) { path, err := pickFile(paths, filters, sortOrder, cache, formats) if err != nil { @@ -351,7 +294,7 @@ func pathCount(path string) (uint32, uint32, error) { return files, directories, nil } -func scanPath(path string, files *files, filters *filters, stats *scanStats, concurrency *concurrency, formats *types.Types) error { +func scanPath(path string, files *files, stats *scanStats, concurrency *concurrency, formats *types.Types) error { var wg sync.WaitGroup err := filepath.WalkDir(path, func(p string, info os.DirEntry, err error) error { @@ -378,10 +321,15 @@ func scanPath(path string, files *files, filters *filters, stats *scanStats, con fmt.Println(err) } - err = appendPaths(path, files, filters, stats, formats) - if err != nil { - fmt.Println(err) + if !formats.Validate(path) { + stats.filesSkipped.Add(1) + + return } + + files.append(path) + + stats.filesMatched.Add(1) }() case info.IsDir(): files, directories, err := pathCount(p) @@ -412,16 +360,10 @@ func scanPath(path string, files *files, filters *filters, stats *scanStats, con return nil } -func fileList(paths []string, filters *filters, sort string, cache *fileCache, formats *types.Types) ([]string, bool) { - if Cache && filters.isEmpty() && !cache.isEmpty() { - return cache.List(), true - } - - var fileList []string - +func scanPaths(paths []string, sort string, cache *fileCache, formats *types.Types) []string { files := &files{ mutex: sync.RWMutex{}, - list: make(map[string][]string), + list: []string{}, } stats := &scanStats{ @@ -451,7 +393,7 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f wg.Done() }() - err := scanPath(paths[i], files, filters, stats, concurrency, formats) + err := scanPath(paths[i], files, stats, concurrency, formats) if err != nil { fmt.Println(err) } @@ -460,10 +402,9 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f wg.Wait() - fileList = prepareDirectories(files, sort) - if stats.filesMatched.Load() < 1 { - return []string{}, false + fmt.Println("No files matched") + return []string{} } if Verbose { @@ -477,35 +418,32 @@ func fileList(paths []string, filters *filters, sort string, cache *fileCache, f ) } - if Cache && filters.isEmpty() { - cache.set(fileList) - } - - return fileList, false + return files.list } -func prepareDirectories(files *files, sort string) []string { - directories := []string{} - - keys := make([]string, len(files.list)) - - i := 0 - for k := range files.list { - keys[i] = k - i++ +func fileList(paths []string, filters *filters, sort string, cache *fileCache, formats *types.Types) []string { + switch { + case Cache && !cache.isEmpty() && filters.isEmpty(): + return cache.List() + case Cache && !cache.isEmpty() && !filters.isEmpty(): + return filters.apply(cache.List()) + case Cache && cache.isEmpty() && !filters.isEmpty(): + cache.set(scanPaths(paths, sort, cache, formats)) + return filters.apply(cache.List()) + case Cache && cache.isEmpty() && filters.isEmpty(): + cache.set(scanPaths(paths, sort, cache, formats)) + return cache.List() + case !Cache && !filters.isEmpty(): + return filters.apply(scanPaths(paths, sort, cache, formats)) + default: + return scanPaths(paths, sort, cache, formats) } - - for i := 0; i < len(keys); i++ { - directories = append(directories, files.list[keys[i]]...) - } - - return directories } func pickFile(args []string, filters *filters, sort string, cache *fileCache, formats *types.Types) (string, error) { - fileList, fromCache := fileList(args, filters, sort, cache, formats) + list := fileList(args, filters, sort, cache, formats) - fileCount := len(fileList) + fileCount := len(list) if fileCount < 1 { return "", ErrNoMediaFound @@ -521,27 +459,7 @@ func pickFile(args []string, filters *filters, sort string, cache *fileCache, fo return "", err } - for i := 0; i < fileCount; i++ { - switch { - case val >= fileCount: - val = 0 - case val < fileCount-1: - val++ - } - - path := fileList[val] - - switch { - case !fromCache && formats.Validate(path): - return path, nil - case !fromCache: - continue - default: - return path, nil - } - } - - return "", ErrNoMediaFound + return list[val], nil } func normalizePath(path string) (string, error) { diff --git a/cmd/filters.go b/cmd/filters.go index f66b695..ffbe45b 100644 --- a/cmd/filters.go +++ b/cmd/filters.go @@ -4,7 +4,10 @@ Copyright © 2023 Seednode package cmd -import "strings" +import ( + "slices" + "strings" +) type filters struct { included []string @@ -30,3 +33,27 @@ func (filters *filters) hasExcludes() bool { func (filters *filters) excludes() string { return strings.Join(filters.excluded, ",") } + +func (filters *filters) apply(fileList []string) []string { + result := make([]string, len(fileList)) + + copy(result, fileList) + + if filters.hasExcludes() { + for _, exclude := range filters.excluded { + result = slices.DeleteFunc(fileList, func(s string) bool { + return strings.Contains(strings.ToLower(s), strings.ToLower(exclude)) + }) + } + } + + if filters.hasIncludes() { + for _, include := range filters.included { + result = slices.DeleteFunc(fileList, func(s string) bool { + return !strings.Contains(strings.ToLower(s), strings.ToLower(include)) + }) + } + } + + return result +} diff --git a/cmd/root.go b/cmd/root.go index 9f191ce..5c52e94 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( ) const ( - ReleaseVersion string = "0.85.1" + ReleaseVersion string = "0.86.0" ) var ( diff --git a/default.pgo b/default.pgo index 4201e08..6e95869 100644 Binary files a/default.pgo and b/default.pgo differ