Remove --page-length and HTML index dump, fix media type identification

This commit is contained in:
Seednode 2024-01-15 08:51:41 -06:00
parent 06c6b70b2c
commit 61060f7895
9 changed files with 59 additions and 271 deletions

View File

@ -37,8 +37,7 @@ The restricted paths are:
- `/debug/pprof/trace`
- `/extensions/available`
- `/extensions/enabled`
- `/index/html`
- `/index/json`
- `/index/`
- `/index/rebuild`
- `/types/available`
- `/types/enabled`
@ -176,7 +175,6 @@ Flags:
-i, --info expose informational endpoints
--max-file-count int skip directories with file counts above this value (default 2147483647)
--min-file-count int skip directories with file counts below this value
--page-length int pagination length for info pages
-p, --port int port to listen on (default 8080)
--prefix string root path for http handlers (for reverse proxying) (default "/")
--profile register net/http/pprof handlers

View File

@ -5,7 +5,6 @@ Copyright © 2024 Seednode <seednode@seedno.de>
package cmd
import (
"bytes"
"embed"
"net/http"
"strconv"
@ -36,12 +35,7 @@ func serveFavicons(errorChannel chan<- error) httprouter.Handle {
return
}
err = w.Header().Write(bytes.NewBufferString("Content-Length: " + strconv.Itoa(len(data))))
if err != nil {
errorChannel <- err
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
_, err = w.Write(data)
if err != nil {

View File

@ -25,10 +25,6 @@ import (
"seedno.de/seednode/roulette/types"
)
var (
filename = regexp.MustCompile(`(.+?)([0-9]*)(\..+)`)
)
type scanStats struct {
filesMatched chan int
filesSkipped chan int
@ -79,14 +75,14 @@ func kill(path string, index *fileIndex) error {
return nil
}
func newFile(list []string, sortOrder string, formats types.Types) (string, error) {
func newFile(list []string, sortOrder string, filename *regexp.Regexp, formats types.Types) (string, error) {
path, err := pickFile(list)
if err != nil {
return "", err
}
if sortOrder == "asc" || sortOrder == "desc" {
splitPath, err := split(path)
splitPath, err := split(path, filename)
if err != nil {
return "", err
}
@ -125,8 +121,8 @@ func newFile(list []string, sortOrder string, formats types.Types) (string, erro
return path, nil
}
func nextFile(filePath, sortOrder string, formats types.Types) (string, error) {
splitPath, err := split(filePath)
func nextFile(filePath, sortOrder string, filename *regexp.Regexp, formats types.Types) (string, error) {
splitPath, err := split(filePath, filename)
if err != nil {
return "", err
}

View File

@ -9,7 +9,6 @@ import (
"fmt"
"net/http"
"os"
"runtime"
"sync"
"time"
@ -213,8 +212,6 @@ func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, enc
time.Since(startTime).Round(time.Microsecond),
)
}
runtime.GC()
}
}

View File

@ -7,194 +7,28 @@ package cmd
import (
"encoding/json"
"fmt"
"io"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/julienschmidt/httprouter"
"github.com/yosssi/gohtml"
"seedno.de/seednode/roulette/types"
)
func paginateIndex(page int, fileCount int, ending bool) string {
var firstPage int = 1
var lastPage int
if fileCount%PageLength == 0 {
lastPage = fileCount / PageLength
} else {
lastPage = (fileCount / PageLength) + 1
}
var prevStatus, nextStatus string = "", ""
if page <= 1 {
prevStatus = " disabled"
}
if page >= lastPage {
nextStatus = " disabled"
}
prevPage := page - 1
if prevPage < 1 {
prevPage = 1
}
nextPage := page + 1
if nextPage > lastPage {
nextPage = fileCount / PageLength
}
var html strings.Builder
if ending {
html.WriteString("<tr><td style=\"border-bottom:none;\">")
} else {
html.WriteString("<tr><td>")
}
html.WriteString(fmt.Sprintf("<button onclick=\"window.location.href = '%s%s/index/html/%d';\">First</button>",
Prefix,
AdminPrefix,
firstPage))
html.WriteString(fmt.Sprintf("<button onclick=\"window.location.href = '%s%s/index/html/%d';\"%s>Prev</button>",
Prefix,
AdminPrefix,
prevPage,
prevStatus))
html.WriteString(fmt.Sprintf("<button onclick=\"window.location.href = '%s%s/index/html/%d';\"%s>Next</button>",
Prefix,
AdminPrefix,
nextPage,
nextStatus))
html.WriteString(fmt.Sprintf("<button onclick=\"window.location.href = '%s%s/index/html/%d';\">Last</button>",
Prefix,
AdminPrefix,
lastPage))
html.WriteString("</td></tr>\n")
return html.String()
}
func serveIndexHtml(args []string, index *fileIndex, shouldPaginate bool) httprouter.Handle {
func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Set("Content-Type", "text/html")
indexDump := index.List()
fileCount := len(indexDump)
var startIndex, stopIndex int
page, err := strconv.Atoi(p.ByName("page"))
if err != nil || page <= 0 {
startIndex = 0
stopIndex = fileCount
} else {
startIndex = ((page - 1) * PageLength)
stopIndex = (startIndex + PageLength)
}
if startIndex > (fileCount - 1) {
indexDump = []string{}
}
if stopIndex > fileCount {
stopIndex = fileCount
}
sort.SliceStable(indexDump, func(p, q int) bool {
return strings.ToLower(indexDump[p]) < strings.ToLower(indexDump[q])
})
var htmlBody strings.Builder
htmlBody.WriteString(`<!DOCTYPE html><html lang="en"><head>`)
htmlBody.WriteString(faviconHtml)
htmlBody.WriteString(`<style>a{text-decoration:none;height:100%;width:100%;color:inherit;cursor:pointer}`)
htmlBody.WriteString(`table,td,tr{border:none;}td{border-bottom:1px solid black;}td{white-space:nowrap;padding:.5em}</style>`)
htmlBody.WriteString(fmt.Sprintf("<title>Index contains %d files</title></head><body><table>", fileCount))
if shouldPaginate && !DisableButtons {
htmlBody.WriteString(paginateIndex(page, fileCount, false))
}
if len(indexDump) > 0 {
for _, v := range indexDump[startIndex:stopIndex] {
var shouldSort = ""
if Sorting {
shouldSort = "?sort=asc"
}
htmlBody.WriteString(fmt.Sprintf("<tr><td><a href=\"%s%s%s%s\">%s</a></td></tr>\n", Prefix, mediaPrefix, v, shouldSort, v))
}
}
if shouldPaginate && !DisableButtons {
htmlBody.WriteString(paginateIndex(page, fileCount, true))
}
htmlBody.WriteString(`</table></body></html>`)
written, err := io.WriteString(w, gohtml.Format(htmlBody.String()))
if err != nil {
return
}
if Verbose {
fmt.Printf("%s | SERVE: HTML index page (%s) to %s in %s\n",
startTime.Format(logDate),
humanReadableSize(written),
realIP(r),
time.Since(startTime).Round(time.Microsecond),
)
}
}
}
func serveIndexJson(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Set("Content-Type", "application/json")
indexedFiles := index.List()
fileCount := len(indexedFiles)
sort.SliceStable(indexedFiles, func(p, q int) bool {
return strings.ToLower(indexedFiles[p]) < strings.ToLower(indexedFiles[q])
})
var startIndex, stopIndex int
page, err := strconv.Atoi(p.ByName("page"))
if err != nil || page <= 0 {
startIndex = 0
stopIndex = fileCount
} else {
startIndex = ((page - 1) * PageLength)
stopIndex = (startIndex + PageLength)
}
if startIndex > (fileCount - 1) {
indexedFiles = []string{}
}
if stopIndex > fileCount {
stopIndex = fileCount
}
response, err := json.MarshalIndent(indexedFiles[startIndex:stopIndex], "", " ")
response, err := json.MarshalIndent(indexDump, "", " ")
if err != nil {
errorChannel <- err
@ -203,6 +37,8 @@ func serveIndexJson(args []string, index *fileIndex, errorChannel chan<- error)
return
}
response = append(response, []byte("\n")...)
written, err := w.Write(response)
if err != nil {
errorChannel <- err
@ -219,35 +55,21 @@ func serveIndexJson(args []string, index *fileIndex, errorChannel chan<- error)
}
}
func serveAvailableExtensions(errorChannel chan<- error) httprouter.Handle {
func serveExtensions(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Set("Content-Type", "text/plain")
written, err := w.Write([]byte(types.SupportedFormats.GetExtensions()))
if err != nil {
errorChannel <- err
var extensions string
if available {
extensions = types.SupportedFormats.GetExtensions()
} else {
extensions = formats.GetExtensions()
}
if Verbose {
fmt.Printf("%s | SERVE: Available extension list (%s) to %s in %s\n",
startTime.Format(logDate),
humanReadableSize(written),
realIP(r),
time.Since(startTime).Round(time.Microsecond),
)
}
}
}
func serveEnabledExtensions(formats types.Types, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Set("Content-Type", "text/plain")
written, err := w.Write([]byte(formats.GetExtensions()))
written, err := w.Write([]byte(extensions))
if err != nil {
errorChannel <- err
}
@ -263,13 +85,21 @@ func serveEnabledExtensions(formats types.Types, errorChannel chan<- error) http
}
}
func serveAvailableMediaTypes(errorChannel chan<- error) httprouter.Handle {
func serveMediaTypes(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Set("Content-Type", "text/plain")
written, err := w.Write([]byte(types.SupportedFormats.GetMediaTypes()))
var mediaTypes string
if available {
mediaTypes = types.SupportedFormats.GetMediaTypes()
} else {
mediaTypes = formats.GetMediaTypes()
}
written, err := w.Write([]byte(mediaTypes))
if err != nil {
errorChannel <- err
}
@ -285,43 +115,13 @@ func serveAvailableMediaTypes(errorChannel chan<- error) httprouter.Handle {
}
}
func serveEnabledMediaTypes(formats types.Types, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Set("Content-Type", "text/plain")
written, err := w.Write([]byte(formats.GetMediaTypes()))
if err != nil {
errorChannel <- err
}
if Verbose {
fmt.Printf("%s | SERVE: Registered media type list (%s) to %s in %s\n",
startTime.Format(logDate),
humanReadableSize(written),
realIP(r),
time.Since(startTime).Round(time.Microsecond),
)
}
}
}
func registerInfoHandlers(mux *httprouter.Router, args []string, index *fileIndex, formats types.Types, errorChannel chan<- error) {
if Index {
registerHandler(mux, Prefix+AdminPrefix+"/index/html", serveIndexHtml(args, index, false))
if PageLength != 0 {
registerHandler(mux, Prefix+AdminPrefix+"/index/html/:page", serveIndexHtml(args, index, true))
registerHandler(mux, Prefix+AdminPrefix+"/index", serveIndex(args, index, errorChannel))
}
registerHandler(mux, Prefix+AdminPrefix+"/index/json", serveIndexJson(args, index, errorChannel))
if PageLength != 0 {
registerHandler(mux, Prefix+AdminPrefix+"/index/json/:page", serveIndexJson(args, index, errorChannel))
}
}
registerHandler(mux, Prefix+AdminPrefix+"/extensions/available", serveAvailableExtensions(errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/extensions/enabled", serveEnabledExtensions(formats, errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/types/available", serveAvailableMediaTypes(errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/types/enabled", serveEnabledMediaTypes(formats, errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/extensions/available", serveExtensions(formats, true, errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/extensions/enabled", serveExtensions(formats, false, errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/types/available", serveMediaTypes(formats, true, errorChannel))
registerHandler(mux, Prefix+AdminPrefix+"/types/enabled", serveMediaTypes(formats, false, errorChannel))
}

View File

@ -17,7 +17,7 @@ import (
const (
AllowedCharacters string = `^[A-z0-9.\-_]+$`
ReleaseVersion string = "5.4.3"
ReleaseVersion string = "6.0.0"
)
var (
@ -47,7 +47,6 @@ var (
Info bool
MaxFileCount int
MinFileCount int
PageLength int
Port int
Prefix string
Profile bool
@ -143,7 +142,6 @@ func init() {
rootCmd.Flags().BoolVarP(&Info, "info", "i", false, "expose informational endpoints")
rootCmd.Flags().IntVar(&MaxFileCount, "max-file-count", math.MaxInt32, "skip directories with file counts above this value")
rootCmd.Flags().IntVar(&MinFileCount, "min-file-count", 0, "skip directories with file counts below this value")
rootCmd.Flags().IntVar(&PageLength, "page-length", 0, "pagination length for info pages")
rootCmd.Flags().IntVarP(&Port, "port", "p", 8080, "port to listen on")
rootCmd.Flags().StringVar(&Prefix, "prefix", "/", "root path for http handlers (for reverse proxying)")
rootCmd.Flags().BoolVar(&Profile, "profile", false, "register net/http/pprof handlers")

View File

@ -6,6 +6,7 @@ package cmd
import (
"fmt"
"regexp"
"sort"
"strings"
@ -38,7 +39,7 @@ func (splitPath *splitPath) decrement() string {
return fmt.Sprintf("%0*d", len(splitPath.number), asInt-1)
}
func split(path string) (*splitPath, error) {
func split(path string, filename *regexp.Regexp) (*splitPath, error) {
split := filename.FindAllStringSubmatch(path, -1)
if len(split) < 1 || len(split[0]) < 3 {
@ -54,8 +55,8 @@ func split(path string) (*splitPath, error) {
return p, nil
}
func getRange(path string, index *fileIndex) (string, string, error) {
splitPath, err := split(path)
func getRange(path string, index *fileIndex, filename *regexp.Regexp) (string, string, error) {
splitPath, err := split(path, filename)
if err != nil {
return "", "", err
}
@ -70,7 +71,7 @@ func getRange(path string, index *fileIndex) (string, string, error) {
Loop:
for _, val := range list {
splitVal, err := split(val)
splitVal, err := split(val, filename)
if err != nil {
return "", "", err
}
@ -94,8 +95,8 @@ func pathUrlEscape(path string) string {
return strings.Replace(path, `'`, `%27`, -1)
}
func paginateSorted(path, first, last, queryParams string, formats types.Types) (string, error) {
split, err := split(path)
func paginate(path, first, last, queryParams string, filename *regexp.Regexp, formats types.Types) (string, error) {
split, err := split(path, filename)
if err != nil {
return "", err
}

View File

@ -13,6 +13,7 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
@ -32,8 +33,6 @@ import (
"seedno.de/seednode/roulette/types/video"
)
var ()
const (
logDate string = `2006-01-02T15:04:05.000-07:00`
sourcePrefix string = `/source`
@ -171,7 +170,7 @@ func serveStaticFile(paths []string, index *fileIndex, errorChannel chan<- error
}
}
func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
func serveRoot(paths []string, index *fileIndex, filename *regexp.Regexp, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
refererUri, err := stripQueryParams(refererToUri(r.Referer()))
if err != nil {
@ -196,7 +195,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z
var path string
if refererUri != "" {
path, err = nextFile(strippedRefererUri, sortOrder, formats)
path, err = nextFile(strippedRefererUri, sortOrder, filename, formats)
if err != nil {
errorChannel <- err
@ -220,7 +219,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z
break loop
}
path, err = newFile(list, sortOrder, formats)
path, err = newFile(list, sortOrder, filename, formats)
switch {
case path == "":
noFiles(w, r)
@ -251,7 +250,7 @@ func serveRoot(paths []string, index *fileIndex, formats types.Types, encoder *z
}
}
func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChannel chan<- error) httprouter.Handle {
func serveMedia(paths []string, index *fileIndex, filename *regexp.Regexp, formats types.Types, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
@ -314,7 +313,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan
return
}
mediaType := format.MediaType(path)
mediaType := format.MediaType(filepath.Ext(path))
fileUri := Prefix + generateFileUri(path)
@ -347,7 +346,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan
var first, last string
if Index && sortOrder != "" {
first, last, err = getRange(path, index)
first, last, err = getRange(path, index, filename)
if err != nil {
errorChannel <- err
@ -358,7 +357,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan
}
if Index && !DisableButtons && sortOrder != "" {
paginate, err := paginateSorted(path, first, last, queryParams, formats)
paginated, err := paginate(path, first, last, queryParams, filename, formats)
if err != nil {
errorChannel <- err
@ -367,7 +366,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan
return
}
htmlBody.WriteString(paginate)
htmlBody.WriteString(paginated)
}
if refreshInterval != "0ms" {
@ -388,7 +387,7 @@ func serveMedia(paths []string, index *fileIndex, formats types.Types, errorChan
formattedPage := gohtml.Format(htmlBody.String())
written, err := io.WriteString(w, formattedPage)
written, err := io.WriteString(w, formattedPage+"\n")
if err != nil {
errorChannel <- err
@ -584,7 +583,9 @@ func ServePage(args []string) error {
return err
}
registerHandler(mux, Prefix, serveRoot(paths, index, formats, encoder, errorChannel))
filename := regexp.MustCompile(`(.+?)([0-9]*)(\..+)`)
registerHandler(mux, Prefix, serveRoot(paths, index, filename, formats, encoder, errorChannel))
Prefix = strings.TrimSuffix(Prefix, "/")
@ -596,7 +597,7 @@ func ServePage(args []string) error {
registerHandler(mux, Prefix+"/favicon.ico", serveFavicons(errorChannel))
registerHandler(mux, Prefix+mediaPrefix+"/*media", serveMedia(paths, index, formats, errorChannel))
registerHandler(mux, Prefix+mediaPrefix+"/*media", serveMedia(paths, index, filename, formats, errorChannel))
registerHandler(mux, Prefix+sourcePrefix+"/*static", serveStaticFile(paths, index, errorChannel))

View File

@ -99,10 +99,13 @@ func (t Types) GetMediaTypes() string {
for _, j := range t {
extensions := j.Extensions()
for _, v := range extensions {
if v != "" {
mediaTypes = append(mediaTypes, v)
}
}
}
mediaTypes = removeDuplicateStr(mediaTypes)