Rework log messages for index rebuilds (scheduled and requested)

This commit is contained in:
Seednode 2024-01-30 14:38:05 -06:00
parent 3814e10f20
commit 70275a734f
3 changed files with 89 additions and 93 deletions

View File

@ -6,11 +6,16 @@ package cmd
import ( import (
"encoding/gob" "encoding/gob"
"encoding/json"
"fmt" "fmt"
"net/http"
"os" "os"
"sort"
"strings"
"sync" "sync"
"time" "time"
"github.com/julienschmidt/httprouter"
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
"seedno.de/seednode/roulette/types" "seedno.de/seednode/roulette/types"
) )
@ -186,6 +191,85 @@ func (index *fileIndex) Import(path string, errorChannel chan<- error) {
} }
} }
func rebuildIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
index.clear()
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
}
func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
if IndexFile != "" {
index.Import(IndexFile, errorChannel)
}
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
}
func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Add("Content-Security-Policy", "default-src 'self';")
w.Header().Set("Content-Type", "application/json;charset=UTF-8")
indexDump := index.List()
sort.SliceStable(indexDump, func(p, q int) bool {
return strings.ToLower(indexDump[p]) < strings.ToLower(indexDump[q])
})
response, err := json.MarshalIndent(indexDump, "", " ")
if err != nil {
errorChannel <- err
serverError(w, r, nil)
return
}
response = append(response, []byte("\n")...)
written, err := w.Write(response)
if err != nil {
errorChannel <- err
}
if Verbose {
fmt.Printf("%s | SERVE: JSON index page (%s) to %s in %s\n",
startTime.Format(logDate),
humanReadableSize(written),
realIP(r),
time.Since(startTime).Round(time.Microsecond),
)
}
}
}
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
if Verbose {
fmt.Printf("%s | SERVE: Index rebuild requested by %s\n",
time.Now().Format(logDate),
realIP(r),
)
}
w.Header().Add("Content-Security-Policy", "default-src 'self';")
w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
rebuildIndex(args, index, formats, encoder, errorChannel)
_, err := w.Write([]byte("Ok\n"))
if err != nil {
errorChannel <- err
return
}
}
}
func registerIndexInterval(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, quit <-chan struct{}, errorChannel chan<- error) { func registerIndexInterval(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, quit <-chan struct{}, errorChannel chan<- error) {
interval, err := time.ParseDuration(IndexInterval) interval, err := time.ParseDuration(IndexInterval)
if err != nil { if err != nil {
@ -200,16 +284,13 @@ func registerIndexInterval(args []string, index *fileIndex, formats types.Types,
for { for {
select { select {
case <-ticker.C: case <-ticker.C:
startTime := time.Now()
rebuildIndex(args, index, formats, encoder, errorChannel)
if Verbose { if Verbose {
fmt.Printf("%s | INDEX: Automatic rebuild took %s\n", fmt.Printf("%s | INDEX: Started scheduled index rebuild\n",
startTime.Format(logDate), time.Now().Format(logDate),
time.Since(startTime).Round(time.Microsecond),
) )
} }
rebuildIndex(args, index, formats, encoder, errorChannel)
case <-quit: case <-quit:
ticker.Stop() ticker.Stop()
@ -218,17 +299,3 @@ func registerIndexInterval(args []string, index *fileIndex, formats types.Types,
} }
}() }()
} }
func rebuildIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
index.clear()
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
}
func importIndex(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) {
if IndexFile != "" {
index.Import(IndexFile, errorChannel)
}
fileList(args, &filters{}, "", index, formats, encoder, errorChannel)
}

View File

@ -5,11 +5,8 @@ Copyright © 2024 Seednode <seednode@seedno.de>
package cmd package cmd
import ( import (
"encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"sort"
"strings"
"time" "time"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
@ -49,74 +46,6 @@ func serveExtensions(formats types.Types, available bool, errorChannel chan<- er
} }
} }
func serveIndex(args []string, index *fileIndex, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Add("Content-Security-Policy", "default-src 'self';")
w.Header().Set("Content-Type", "application/json;charset=UTF-8")
indexDump := index.List()
sort.SliceStable(indexDump, func(p, q int) bool {
return strings.ToLower(indexDump[p]) < strings.ToLower(indexDump[q])
})
response, err := json.MarshalIndent(indexDump, "", " ")
if err != nil {
errorChannel <- err
serverError(w, r, nil)
return
}
response = append(response, []byte("\n")...)
written, err := w.Write(response)
if err != nil {
errorChannel <- err
}
if Verbose {
fmt.Printf("%s | SERVE: JSON index page (%s) to %s in %s\n",
startTime.Format(logDate),
humanReadableSize(written),
realIP(r),
time.Since(startTime).Round(time.Microsecond),
)
}
}
}
func serveIndexRebuild(args []string, index *fileIndex, formats types.Types, encoder *zstd.Encoder, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now()
w.Header().Add("Content-Security-Policy", "default-src 'self';")
w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
rebuildIndex(args, index, formats, encoder, errorChannel)
_, err := w.Write([]byte("Ok\n"))
if err != nil {
errorChannel <- err
return
}
if Verbose {
fmt.Printf("%s | SERVE: Index rebuild requested by %s took %s\n",
startTime.Format(logDate),
realIP(r),
time.Since(startTime).Round(time.Microsecond),
)
}
}
}
func serveMediaTypes(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle { func serveMediaTypes(formats types.Types, available bool, errorChannel chan<- error) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
startTime := time.Now() startTime := time.Now()

View File

@ -17,7 +17,7 @@ import (
const ( const (
AllowedCharacters string = `^[A-z0-9.\-_]+$` AllowedCharacters string = `^[A-z0-9.\-_]+$`
ReleaseVersion string = "8.3.0" ReleaseVersion string = "8.3.1"
) )
var ( var (