SeanOMik
/
roulette
mirror of https://git.seedno.de/seednode/roulette.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Continue scanning instead of aborting on fs.ErrPermission or fs.ErrNotExist; add --debug flag to view output from those errors; initialize error logging earlier in setup

This commit is contained in:
Seednode 2024-01-06 09:41:30 -06:00
parent e9d75ac6f3
commit 6f29f89acf
5 changed files with 84 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -153,9 +153,10 @@ Flags:
--case-sensitive use case-sensitive matching for filters
--code enable support for source code files
--code-theme string theme for source code syntax highlighting (default "solarized-dark256")
--compression string compression format to use for index (flate, gzip, lz4, lzw, none, snappy, zlib, zstd) (default "zstd")
--compression string compression format to use for index (flate, gzip, lz5, lzw, none, snappy, zlib, zstd) (default "zstd")
--compression-fast use fastest compression level (default is best)
--concurrency int maximum concurrency for scan threads (default 8192)
--concurrency int maximum concurrency for scan threads (default 9223372036854775807)
-d, --debug display even more verbose logs
--disable-buttons disable first/prev/next/last buttons
--exit-on-error shut down webserver on error, instead of just printing error
--fallback serve files as application/octet-stream if no matching format is registered

147
cmd/files.go
View File

@ -240,74 +240,74 @@ func hasSupportedFiles(path string, formats types.Types) (bool, error) {
}
}
func walkPath(path string, fileChannel chan<- string, stats *scanStats, limit chan struct{}, formats types.Types) error {
func walkPath(path string, fileChannel chan<- string, wg0 *sync.WaitGroup, stats *scanStats, limit chan struct{}, formats types.Types, errorChannel chan<- error) {
defer func() {
wg0.Done()
}()
limit <- struct{}{}
defer func() {
<-limit
}()
errorChannel := make(chan error)
done := make(chan bool)
nodes, err := os.ReadDir(path)
if err != nil {
return err
stats.directoriesSkipped <- 1
errorChannel <- err
return
}
var directories, files = 0, 0
var files = 0
var skipDir = false
for _, node := range nodes {
if node.IsDir() {
directories++
} else {
if Ignore && node.Name() == IgnoreFile {
skipDir = true
}
files++
if Ignore && !node.IsDir() && node.Name() == IgnoreFile {
skipDir = true
}
files++
}
var skipFiles = false
if files <= MaxFileCount && files >= MinFileCount && !skipDir {
stats.directoriesMatched <- 1
} else {
if files > MaxFileCount || files < MinFileCount || skipDir {
stats.filesSkipped <- files
stats.directoriesSkipped <- 1
skipFiles = true
} else {
stats.directoriesMatched <- 1
}
var wg sync.WaitGroup
var wg1 sync.WaitGroup
wg.Add(1)
wg1.Add(1)
go func() {
defer wg.Done()
defer wg1.Done()
for _, node := range nodes {
wg.Add(1)
wg1.Add(1)
go func(node fs.DirEntry) {
defer wg.Done()
defer wg1.Done()
fullPath := filepath.Join(path, node.Name())
switch {
case node.IsDir() && Recursive:
err := walkPath(fullPath, fileChannel, stats, limit, formats)
if err != nil {
errorChannel <- err
wg0.Add(1)
return
}
walkPath(fullPath, fileChannel, wg0, stats, limit, formats, errorChannel)
case !node.IsDir() && !skipFiles:
path, err := normalizePath(fullPath)
if err != nil {
errorChannel <- err
stats.filesSkipped <- 1
return
}
@ -325,35 +325,16 @@ func walkPath(path string, fileChannel chan<- string, stats *scanStats, limit ch
}
}()
go func() {
wg.Wait()
time.Sleep(1 * time.Microsecond)
close(done)
}()
Poll:
for {
select {
case err := <-errorChannel:
return err
case <-done:
break Poll
}
}
return nil
wg1.Wait()
}
func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types) ([]string, error) {
func scanPaths(paths []string, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
startTime := time.Now()
var filesMatched, filesSkipped int
var directoriesMatched, directoriesSkipped int
fileChannel := make(chan string)
errorChannel := make(chan error)
done := make(chan bool)
stats := &scanStats{
@ -422,41 +403,19 @@ func scanPaths(paths []string, sort string, index *fileIndex, formats types.Type
limit := make(chan struct{}, Concurrency)
var wg sync.WaitGroup
var wg0 sync.WaitGroup
for i := 0; i < len(paths); i++ {
wg.Add(1)
wg0.Add(1)
go func(i int) {
defer func() {
wg.Done()
}()
err := walkPath(paths[i], fileChannel, stats, limit, formats)
if err != nil {
errorChannel <- err
return
}
walkPath(paths[i], fileChannel, &wg0, stats, limit, formats, errorChannel)
}(i)
}
go func() {
wg.Wait()
wg0.Wait()
close(done)
}()
Poll:
for {
select {
case err := <-errorChannel:
return []string{}, err
case <-done:
break Poll
}
}
close(done)
if Verbose {
fmt.Printf("%s | INDEX: Selected %d/%d files across %d/%d directories in %s\n",
@ -471,45 +430,27 @@ Poll:
slices.Sort(list)
return list, nil
return list
}
func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types) ([]string, error) {
func fileList(paths []string, filters *filters, sort string, index *fileIndex, formats types.Types, errorChannel chan<- error) []string {
switch {
case Index && !index.isEmpty() && filters.isEmpty():
return index.List(), nil
return index.List()
case Index && !index.isEmpty() && !filters.isEmpty():
return filters.apply(index.List()), nil
return filters.apply(index.List())
case Index && index.isEmpty() && !filters.isEmpty():
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}
index.set(list)
index.set(scanPaths(paths, sort, index, formats, errorChannel))
return filters.apply(index.List()), nil
return filters.apply(index.List())
case Index && index.isEmpty() && filters.isEmpty():
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}
index.set(list)
index.set(scanPaths(paths, sort, index, formats, errorChannel))
return index.List(), nil
return index.List()
case !Index && !filters.isEmpty():
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}
return filters.apply(list), nil
return filters.apply(scanPaths(paths, sort, index, formats, errorChannel))
default:
list, err := scanPaths(paths, sort, index, formats)
if err != nil {
return []string{}, err
}
return list, nil
return scanPaths(paths, sort, index, formats, errorChannel)
}
}

18
cmd/index.go
View File

@ -266,12 +266,7 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
index.clear()
_, err := fileList(args, &filters{}, "", index, formats)
if err != nil {
errorChannel <- err
return
}
fileList(args, &filters{}, "", index, formats, errorChannel)
w.Header().Set("Content-Type", "text/plain")
@ -279,18 +274,13 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, err
}
}
func importIndex(args []string, index *fileIndex, formats types.Types) error {
func importIndex(args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
if IndexFile != "" {
err := index.Import(IndexFile)
if err == nil {
return nil
return
}
}
_, err := fileList(args, &filters{}, "", index, formats)
if err != nil {
return err
}
return nil
fileList(args, &filters{}, "", index, formats, errorChannel)
}

8
cmd/root.go
View File

@ -15,7 +15,7 @@ import (
const (
AllowedCharacters string = `^[A-z0-9.\-_]+$`
ReleaseVersion string = "3.12.1"
ReleaseVersion string = "4.0.0"
)
var (
@ -31,6 +31,7 @@ var (
Compression string
CompressionFast bool
Concurrency int
Debug bool
DisableButtons bool
ExitOnError bool
Fallback bool
@ -138,7 +139,8 @@ func init() {
rootCmd.Flags().StringVar(&CodeTheme, "code-theme", "solarized-dark256", "theme for source code syntax highlighting")
rootCmd.Flags().StringVar(&Compression, "compression", "zstd", "compression format to use for index (flate, gzip, lz5, lzw, none, snappy, zlib, zstd)")
rootCmd.Flags().BoolVar(&CompressionFast, "compression-fast", false, "use fastest compression level (default is best)")
rootCmd.Flags().IntVar(&Concurrency, "concurrency", 8192, "maximum concurrency for scan threads")
rootCmd.Flags().IntVar(&Concurrency, "concurrency", int(^uint(0)>>1), "maximum concurrency for scan threads")
rootCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "display even more verbose logs")
rootCmd.Flags().BoolVar(&DisableButtons, "disable-buttons", false, "disable first/prev/next/last buttons")
rootCmd.Flags().BoolVar(&ExitOnError, "exit-on-error", false, "shut down webserver on error, instead of just printing error")
rootCmd.Flags().BoolVar(&Fallback, "fallback", false, "serve files as application/octet-stream if no matching format is registered")
@ -172,6 +174,8 @@ func init() {
rootCmd.Flags().SetInterspersed(true)
rootCmd.MarkFlagsMutuallyExclusive("debug", "exit-on-error")
rootCmd.MarkFlagsOneRequired(RequiredArgs...)
rootCmd.SetHelpCommand(&cobra.Command{

51
cmd/web.go
View File

@ -206,14 +206,7 @@ func serveRoot(paths []string, regexes *regexes, index *fileIndex, formats types
}
}
list, err := fileList(paths, filters, sortOrder, index, formats)
if err != nil {
errorChannel <- err
serverError(w, r, nil)
return
}
list := fileList(paths, filters, sortOrder, index, formats, errorChannel)
loop:
for timeout := time.After(timeout); ; {
@ -552,6 +545,31 @@ func ServePage(args []string) error {
errorChannel := make(chan error)
go func() {
for err := range errorChannel {
prefix := "ERROR"
switch {
case errors.Is(err, os.ErrNotExist) && Debug:
prefix = "DEBUG"
case errors.Is(err, os.ErrNotExist):
continue
case errors.Is(err, os.ErrPermission) && Debug:
prefix = "DEBUG"
case errors.Is(err, os.ErrPermission):
continue
}
fmt.Printf("%s | %s: %v\n", time.Now().Format(logDate), prefix, err)
if ExitOnError {
fmt.Printf("%s | %s: Shutting down...\n", time.Now().Format(logDate), prefix)
srv.Shutdown(context.Background())
}
}
}()
registerHandler(mux, Prefix, serveRoot(paths, regexes, index, formats, errorChannel))
Prefix = strings.TrimSuffix(Prefix, "/")
@ -573,10 +591,7 @@ func ServePage(args []string) error {
if Index {
registerHandler(mux, Prefix+AdminPrefix+"/index/rebuild", serveIndexRebuild(args, index, formats, errorChannel))
err = importIndex(paths, index, formats)
if err != nil {
return err
}
importIndex(paths, index, formats, errorChannel)
}
if Info {
@ -591,18 +606,6 @@ func ServePage(args []string) error {
fmt.Printf("WARNING! Files *will* be deleted after serving!\n\n")
}
go func() {
for err := range errorChannel {
fmt.Printf("%s | ERROR: %v\n", time.Now().Format(logDate), err)
if ExitOnError {
fmt.Printf("%s | ERROR: Shutting down...\n", time.Now().Format(logDate))
srv.Shutdown(context.Background())
}
}
}()
if Verbose {
fmt.Printf("%s | SERVE: Listening on http://%s%s/\n",
time.Now().Format(logDate),