From 61060f7895a9d3f046d7e7877c3c1ae925357b77 Mon Sep 17 00:00:00 2001 From: Seednode Date: Mon, 15 Jan 2024 08:51:41 -0600 Subject: [PATCH] Remove --page-length and HTML index dump, fix media type identification --- README.md | 4 +- cmd/favicons.go | 8 +- cmd/files.go | 12 +-- cmd/index.go | 3 - cmd/info.go | 254 +++++------------------------------------------- cmd/root.go | 4 +- cmd/sort.go | 13 +-- cmd/web.go | 27 ++--- types/types.go | 5 +- 9 files changed, 59 insertions(+), 271 deletions(-) diff --git a/README.md b/README.md index e15cd4f..5d8b618 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,7 @@ The restricted paths are: - `/debug/pprof/trace` - `/extensions/available` - `/extensions/enabled` -- `/index/html` -- `/index/json` +- `/index/` - `/index/rebuild` - `/types/available` - `/types/enabled` @@ -176,7 +175,6 @@ Flags: -i, --info expose informational endpoints --max-file-count int skip directories with file counts above this value (default 2147483647) --min-file-count int skip directories with file counts below this value - --page-length int pagination length for info pages -p, --port int port to listen on (default 8080) --prefix string root path for http handlers (for reverse proxying) (default "/") --profile register net/http/pprof handlers diff --git a/cmd/favicons.go b/cmd/favicons.go index fd5b0bf..dc8e412 100644 --- a/cmd/favicons.go +++ b/cmd/favicons.go @@ -5,7 +5,6 @@ Copyright © 2024 Seednode package cmd import ( - "bytes" "embed" "net/http" "strconv" @@ -36,12 +35,7 @@ func serveFavicons(errorChannel chan<- error) httprouter.Handle { return } - err = w.Header().Write(bytes.NewBufferString("Content-Length: " + strconv.Itoa(len(data)))) - if err != nil { - errorChannel <- err - - return - } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) _, err = w.Write(data) if err != nil { diff --git a/cmd/files.go b/cmd/files.go index 27488ec..8be6be0 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -25,10 +25,6 @@ import ( "seedno.de/seednode/roulette/types" ) -var ( - filename = regexp.MustCompile(`(.+?)([0-9]*)(\..+)`) -) - type scanStats struct { filesMatched chan int filesSkipped chan int @@ -79,14 +75,14 @@ func kill(path string, index *fileIndex) error { return nil } -func newFile(list []string, sortOrder string, formats types.Types) (string, error) { +func newFile(list []string, sortOrder string, filename *regexp.Regexp, formats types.Types) (string, error) { path, err := pickFile(list) if err != nil { return "", err } if sortOrder == "asc" || sortOrder == "desc" { - splitPath, err := split(path) + splitPath, err := split(path, filename) if err != nil { return "", err } @@ -125,8 +121,8 @@ func newFile(list []string, sortOrder string, formats types.Types) (string, erro return path, nil } -func nextFile(filePath, sortOrder string, formats types.Types) (string, error) { - splitPath, err := split(filePath) +func nextFile(filePath, sortOrder string, filename *regexp.Regexp, formats types.Types) (string, error) { + splitPath, err := split(filePath, filename) if err != nil { return "", err } diff --git a/cmd/index.go b/cmd/index.go index 593bf11..3367bf9 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -9,7 +9,6 @@ import ( "fmt" "net/http" "os" - "runtime" "sync" "time" @@ -213,8 +212,6 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, enc time.Since(startTime).Round(time.Microsecond), ) } - - runtime.GC() } } diff --git a/cmd/info.go b/cmd/info.go index 34c8ed5..72aed70 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -7,194 +7,28 @@ package cmd import ( "encoding/json" "fmt" - "io" "net/http" "sort" - "strconv" "strings" "time" "github.com/julienschmidt/httprouter" - "github.com/yosssi/gohtml" "seedno.de/seednode/roulette/types" ) -func paginateIndex(page int, fileCount int, ending bool) string { - var firstPage int = 1 - var lastPage int - - if fileCount%PageLength == 0 { - lastPage = fileCount / PageLength - } else { - lastPage = (fileCount / PageLength) + 1 - } - - var prevStatus, nextStatus string = "", "" - - if page <= 1 { - prevStatus = " disabled" - } - - if page >= lastPage { - nextStatus = " disabled" - } - - prevPage := page - 1 - if prevPage < 1 { - prevPage = 1 - } - - nextPage := page + 1 - if nextPage > lastPage { - nextPage = fileCount / PageLength - } - - var html strings.Builder - - if ending { - html.WriteString("") - } else { - html.WriteString("") - } - - html.WriteString(fmt.Sprintf("", - Prefix, - AdminPrefix, - firstPage)) - - html.WriteString(fmt.Sprintf("", - Prefix, - AdminPrefix, - prevPage, - prevStatus)) - - html.WriteString(fmt.Sprintf("", - Prefix, - AdminPrefix, - nextPage, - nextStatus)) - - html.WriteString(fmt.Sprintf("", - Prefix, - AdminPrefix, - lastPage)) - - html.WriteString("\n") - - return html.String() -} - -func serveIndexHtml(args []string, index *fileIndex, shouldPaginate bool) httprouter.Handle { +func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { startTime := time.Now() - w.Header().Set("Content-Type", "text/html") - indexDump := index.List() - fileCount := len(indexDump) - - var startIndex, stopIndex int - - page, err := strconv.Atoi(p.ByName("page")) - if err != nil || page <= 0 { - startIndex = 0 - stopIndex = fileCount - } else { - startIndex = ((page - 1) * PageLength) - stopIndex = (startIndex + PageLength) - } - - if startIndex > (fileCount - 1) { - indexDump = []string{} - } - - if stopIndex > fileCount { - stopIndex = fileCount - } - sort.SliceStable(indexDump, func(p, q int) bool { return strings.ToLower(indexDump[p]) < strings.ToLower(indexDump[q]) }) - var htmlBody strings.Builder - htmlBody.WriteString(``) - htmlBody.WriteString(faviconHtml) - htmlBody.WriteString(``) - htmlBody.WriteString(fmt.Sprintf("Index contains %d files", fileCount)) - - if shouldPaginate && !DisableButtons { - htmlBody.WriteString(paginateIndex(page, fileCount, false)) - } - - if len(indexDump) > 0 { - for _, v := range indexDump[startIndex:stopIndex] { - var shouldSort = "" - - if Sorting { - shouldSort = "?sort=asc" - } - htmlBody.WriteString(fmt.Sprintf("\n", Prefix, mediaPrefix, v, shouldSort, v)) - } - } - - if shouldPaginate && !DisableButtons { - htmlBody.WriteString(paginateIndex(page, fileCount, true)) - } - - htmlBody.WriteString(`
%s
`) - - written, err := io.WriteString(w, gohtml.Format(htmlBody.String())) - if err != nil { - return - } - - if Verbose { - fmt.Printf("%s | SERVE: HTML index page (%s) to %s in %s\n", - startTime.Format(logDate), - humanReadableSize(written), - realIP(r), - time.Since(startTime).Round(time.Microsecond), - ) - } - } -} - -func serveIndexJson(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle { - return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { - startTime := time.Now() - w.Header().Set("Content-Type", "application/json") - indexedFiles := index.List() - - fileCount := len(indexedFiles) - - sort.SliceStable(indexedFiles, func(p, q int) bool { - return strings.ToLower(indexedFiles[p]) < strings.ToLower(indexedFiles[q]) - }) - - var startIndex, stopIndex int - - page, err := strconv.Atoi(p.ByName("page")) - if err != nil || page <= 0 { - startIndex = 0 - stopIndex = fileCount - } else { - startIndex = ((page - 1) * PageLength) - stopIndex = (startIndex + PageLength) - } - - if startIndex > (fileCount - 1) { - indexedFiles = []string{} - } - - if stopIndex > fileCount { - stopIndex = fileCount - } - - response, err := json.MarshalIndent(indexedFiles[startIndex:stopIndex], "", " ") + response, err := json.MarshalIndent(indexDump, "", " ") if err != nil { errorChannel <- err @@ -203,6 +37,8 @@ func serveIndexJson(args []string, index *fileIndex, errorChannel chan<- error) return } + response = append(response, []byte("\n")...) + written, err := w.Write(response) if err != nil { errorChannel <- err @@ -219,35 +55,21 @@ func serveIndexJson(args []string, index *fileIndex, errorChannel chan<- error) } } -func serveAvailableExtensions(errorChannel chan<- error) httprouter.Handle { +func serveExtensions(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { startTime := time.Now() w.Header().Set("Content-Type", "text/plain") - written, err := w.Write([]byte(types.SupportedFormats.GetExtensions())) - if err != nil { - errorChannel <- err + var extensions string + + if available { + extensions = types.SupportedFormats.GetExtensions() + } else { + extensions = formats.GetExtensions() } - if Verbose { - fmt.Printf("%s | SERVE: Available extension list (%s) to %s in %s\n", - startTime.Format(logDate), - humanReadableSize(written), - realIP(r), - time.Since(startTime).Round(time.Microsecond), - ) - } - } -} - -func serveEnabledExtensions(formats types.Types, errorChannel chan<- error) httprouter.Handle { - return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { - startTime := time.Now() - - w.Header().Set("Content-Type", "text/plain") - - written, err := w.Write([]byte(formats.GetExtensions())) + written, err := w.Write([]byte(extensions)) if err != nil { errorChannel <- err } @@ -263,13 +85,21 @@ func serveEnabledExtensions(formats types.Types, errorChannel chan<- error) http } } -func serveAvailableMediaTypes(errorChannel chan<- error) httprouter.Handle { +func serveMediaTypes(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { startTime := time.Now() w.Header().Set("Content-Type", "text/plain") - written, err := w.Write([]byte(types.SupportedFormats.GetMediaTypes())) + var mediaTypes string + + if available { + mediaTypes = types.SupportedFormats.GetMediaTypes() + } else { + mediaTypes = formats.GetMediaTypes() + } + + written, err := w.Write([]byte(mediaTypes)) if err != nil { errorChannel <- err } @@ -285,43 +115,13 @@ func serveAvailableMediaTypes(errorChannel chan<- error) httprouter.Handle { } } -func serveEnabledMediaTypes(formats types.Types, errorChannel chan<- error) httprouter.Handle { - return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { - startTime := time.Now() - - w.Header().Set("Content-Type", "text/plain") - - written, err := w.Write([]byte(formats.GetMediaTypes())) - if err != nil { - errorChannel <- err - } - - if Verbose { - fmt.Printf("%s | SERVE: Registered media type list (%s) to %s in %s\n", - startTime.Format(logDate), - humanReadableSize(written), - realIP(r), - time.Since(startTime).Round(time.Microsecond), - ) - } - } -} - func registerInfoHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) { if Index { - registerHandler(mux, Prefix+AdminPrefix+"/index/html", serveIndexHtml(args, index, false)) - if PageLength != 0 { - registerHandler(mux, Prefix+AdminPrefix+"/index/html/:page", serveIndexHtml(args, index, true)) - } - - registerHandler(mux, Prefix+AdminPrefix+"/index/json", serveIndexJson(args, index, errorChannel)) - if PageLength != 0 { - registerHandler(mux, Prefix+AdminPrefix+"/index/json/:page", serveIndexJson(args, index, errorChannel)) - } + registerHandler(mux, Prefix+AdminPrefix+"/index", serveIndex(args, index, errorChannel)) } - registerHandler(mux, Prefix+AdminPrefix+"/extensions/available", serveAvailableExtensions(errorChannel)) - registerHandler(mux, Prefix+AdminPrefix+"/extensions/enabled", serveEnabledExtensions(formats, errorChannel)) - registerHandler(mux, Prefix+AdminPrefix+"/types/available", serveAvailableMediaTypes(errorChannel)) - registerHandler(mux, Prefix+AdminPrefix+"/types/enabled", serveEnabledMediaTypes(formats, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/extensions/available", serveExtensions(formats, true, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/extensions/enabled", serveExtensions(formats, false, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/types/available", serveMediaTypes(formats, true, errorChannel)) + registerHandler(mux, Prefix+AdminPrefix+"/types/enabled", serveMediaTypes(formats, false, errorChannel)) } diff --git a/cmd/root.go b/cmd/root.go index 59f88f1..24c3dda 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( const ( AllowedCharacters string = `^[A-z0-9.\-_]+$` - ReleaseVersion string = "5.4.3" + ReleaseVersion string = "6.0.0" ) var ( @@ -47,7 +47,6 @@ var ( Info bool MaxFileCount int MinFileCount int - PageLength int Port int Prefix string Profile bool @@ -143,7 +142,6 @@ func init() { rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints") rootCmd.Flags().IntVar(&MaxFileCount, "max-file-count", math.MaxInt32, "skip directories with file counts above this value") rootCmd.Flags().IntVar(&MinFileCount, "min-file-count", 0, "skip directories with file counts below this value") - rootCmd.Flags().IntVar(&PageLength, "page-length", 0, "pagination length for info pages") rootCmd.Flags().IntVarP(&Port, "port", "p", 8080, "port to listen on") rootCmd.Flags().StringVar(&Prefix, "prefix", "/", "root path for http handlers (for reverse proxying)") rootCmd.Flags().BoolVar(&Profile, "profile", false, "register net/http/pprof handlers") diff --git a/cmd/sort.go b/cmd/sort.go index b2d9280..acf04d9 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "regexp" "sort" "strings" @@ -38,7 +39,7 @@ func (splitPath *splitPath) decrement() string { return fmt.Sprintf("%0*d", len(splitPath.number), asInt-1) } -func split(path string) (*splitPath, error) { +func split(path string, filename *regexp.Regexp) (*splitPath, error) { split := filename.FindAllStringSubmatch(path, -1) if len(split) < 1 || len(split[0]) < 3 { @@ -54,8 +55,8 @@ func split(path string) (*splitPath, error) { return p, nil } -func getRange(path string, index *fileIndex) (string, string, error) { - splitPath, err := split(path) +func getRange(path string, index *fileIndex, filename *regexp.Regexp) (string, string, error) { + splitPath, err := split(path, filename) if err != nil { return "", "", err } @@ -70,7 +71,7 @@ func getRange(path string, index *fileIndex) (string, string, error) { Loop: for _, val := range list { - splitVal, err := split(val) + splitVal, err := split(val, filename) if err != nil { return "", "", err } @@ -94,8 +95,8 @@ func pathUrlEscape(path string) string { return strings.Replace(path, `'`, `%27`, -1) } -func paginateSorted(path, first, last, queryParams string, formats types.Types) (string, error) { - split, err := split(path) +func paginate(path, first, last, queryParams string, filename *regexp.Regexp, formats types.Types) (string, error) { + split, err := split(path, filename) if err != nil { return "", err } diff --git a/cmd/web.go b/cmd/web.go index c4ff93d..13c4cae 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "runtime" "strconv" "strings" @@ -32,8 +33,6 @@ import ( "seedno.de/seednode/roulette/types/video" ) -var () - const ( logDate string = `2006-01-02T15:04:05.000-07:00` sourcePrefix string = `/source` @@ -171,7 +170,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error } } -func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { +func serveRoot(paths []string, index *fileIndex, filename *regexp.Regexp, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { refererUri, err := stripQueryParams(refererToUri(r.Referer())) if err != nil { @@ -196,7 +195,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z var path string if refererUri != "" { - path, err = nextFile(strippedRefererUri, sortOrder, formats) + path, err = nextFile(strippedRefererUri, sortOrder, filename, formats) if err != nil { errorChannel <- err @@ -220,7 +219,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z break loop } - path, err = newFile(list, sortOrder, formats) + path, err = newFile(list, sortOrder, filename, formats) switch { case path == "": noFiles(w, r) @@ -251,7 +250,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z } } -func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle { +func serveMedia(paths []string, index *fileIndex, filename *regexp.Regexp, formats types.Types, errorChannel chan<- error) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { startTime := time.Now() @@ -314,7 +313,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan return } - mediaType := format.MediaType(path) + mediaType := format.MediaType(filepath.Ext(path)) fileUri := Prefix + generateFileUri(path) @@ -347,7 +346,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan var first, last string if Index && sortOrder != "" { - first, last, err = getRange(path, index) + first, last, err = getRange(path, index, filename) if err != nil { errorChannel <- err @@ -358,7 +357,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan } if Index && !DisableButtons && sortOrder != "" { - paginate, err := paginateSorted(path, first, last, queryParams, formats) + paginated, err := paginate(path, first, last, queryParams, filename, formats) if err != nil { errorChannel <- err @@ -367,7 +366,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan return } - htmlBody.WriteString(paginate) + htmlBody.WriteString(paginated) } if refreshInterval != "0ms" { @@ -388,7 +387,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan formattedPage := gohtml.Format(htmlBody.String()) - written, err := io.WriteString(w, formattedPage) + written, err := io.WriteString(w, formattedPage+"\n") if err != nil { errorChannel <- err @@ -584,7 +583,9 @@ func ServePage(args []string) error { return err } - registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel)) + filename := regexp.MustCompile(`(.+?)([0-9]*)(\..+)`) + + registerHandler(mux, Prefix, serveRoot(paths, index, filename, formats, encoder, errorChannel)) Prefix = strings.TrimSuffix(Prefix, "/") @@ -596,7 +597,7 @@ func ServePage(args []string) error { registerHandler(mux, Prefix+"/favicon.ico", serveFavicons(errorChannel)) - registerHandler(mux, Prefix+mediaPrefix+"/*media", serveMedia(paths, index, formats, errorChannel)) + registerHandler(mux, Prefix+mediaPrefix+"/*media", serveMedia(paths, index, filename, formats, errorChannel)) registerHandler(mux, Prefix+sourcePrefix+"/*static", serveStaticFile(paths, index, errorChannel)) diff --git a/types/types.go b/types/types.go index 2254f0f..83bf08f 100644 --- a/types/types.go +++ b/types/types.go @@ -99,8 +99,11 @@ func (t Types) GetMediaTypes() string { for _, j := range t { extensions := j.Extensions() + for _, v := range extensions { - mediaTypes = append(mediaTypes, v) + if v != "" { + mediaTypes = append(mediaTypes, v) + } } }