cmd/index.go

@@ -24,17 +24,6 @@ import (
 	"seedno.de/seednode/roulette/types"
 )
 
-var CompressionFormats = []string{
-	"flate",
-	"gzip",
-	"lz4",
-	"lzw",
-	"none",
-	"snappy",
-	"zlib",
-	"zstd",
-}
-
 type fileIndex struct {
 	mutex *sync.RWMutex
 	list  []string
@@ -113,7 +102,7 @@ func getReader(format string, file io.Reader) (io.Reader, error) {
 	case "lz4":
 		return lz4.NewReader(file), nil
 	case "lzw":
-		return lzw.NewReader(file, lzw.MSB, 8), nil
+		return lzw.NewReader(file, lzw.LSB, 8), nil
 	case "none":
 		return io.NopCloser(file), nil
 	case "snappy":
@@ -156,7 +145,7 @@ func getWriter(format string, file io.WriteCloser) (io.WriteCloser, error) {
 
 		return encoder, nil
 	case format == "lzw":
-		return lzw.NewWriter(file, lzw.MSB, 8), nil
+		return lzw.NewWriter(file, lzw.LSB, 8), nil
 	case format == "none":
 		return file, nil
 	case format == "snappy":

cmd/root.go

@@ -17,7 +17,7 @@ import (
 
 const (
 	AllowedCharacters string = `^[A-z0-9.\-_]+$`
-	ReleaseVersion    string = "4.1.0"
+	ReleaseVersion    string = "4.0.4"
 )
 
 var (
@@ -63,6 +63,17 @@ var (
 	Version         bool
 	Videos          bool
 
+	CompressionFormats = []string{
+		"flate",
+		"gzip",
+		"lz4",
+		"lzw",
+		"none",
+		"snappy",
+		"zlib",
+		"zstd",
+	}
+
 	RequiredArgs = []string{
 		"all",
 		"audio",

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/klauspost/compress v1.17.4
-	github.com/pierrec/lz4/v4 v4.1.21
+	github.com/pierrec/lz4/v4 v4.1.19
 	github.com/spf13/cobra v1.8.0
 	github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4
 	golang.org/x/image v0.15.0
@@ -17,5 +17,5 @@ require (
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
 )

go.sum

@@ -17,8 +17,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
 github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
-github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4=
-github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
@@ -28,7 +28,7 @@ github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 h1:0sw0nJM544SpsihWx
 github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4/go.mod h1:+ccdNT0xMY1dtc5XBxumbYfOUhmduiGudqaDgD2rVRE=
 golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
 golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

vendor/github.com/pierrec/lz4/v4/README.md

@@ -21,7 +21,7 @@ go get github.com/pierrec/lz4/v4
 There is a command line interface tool to compress and decompress LZ4 files.
 
 ```
-go install github.com/pierrec/lz4/v4/cmd/lz4c@latest
+go install github.com/pierrec/lz4/v4/cmd/lz4c
 ```
 
 Usage

vendor/github.com/pierrec/lz4/v4/compressing_reader.go

@@ -1,222 +0,0 @@
-package lz4
-
-import (
-	"errors"
-	"io"
-
-	"github.com/pierrec/lz4/v4/internal/lz4block"
-	"github.com/pierrec/lz4/v4/internal/lz4errors"
-	"github.com/pierrec/lz4/v4/internal/lz4stream"
-)
-
-type crState int
-
-const (
-	crStateInitial crState = iota
-	crStateReading 
-	crStateFlushing
-	crStateDone
-)
-
-type CompressingReader struct {
-	state crState
-	src io.ReadCloser // source reader
-	level lz4block.CompressionLevel // how hard to try
-	frame *lz4stream.Frame // frame being built
-	in []byte
-	out ovWriter
-	handler func(int)
-}
-
-// NewCompressingReader creates a reader which reads compressed data from
-// raw stream. This makes it a logical opposite of a normal lz4.Reader.
-// We require an io.ReadCloser as an underlying source for compatibility
-// with Go's http.Request.
-func NewCompressingReader(src io.ReadCloser) *CompressingReader {
-	zrd := &CompressingReader {
-		frame: lz4stream.NewFrame(),
-	}
-
-	_ = zrd.Apply(DefaultBlockSizeOption, DefaultChecksumOption, defaultOnBlockDone)
-	zrd.Reset(src)
-
-	return zrd
-}
-
-// Source exposes the underlying source stream for introspection and control.
-func (zrd *CompressingReader) Source() io.ReadCloser {
-	return zrd.src
-}
-
-// Close simply invokes the underlying stream Close method. This method is
-// provided for the benefit of Go http client/server, which relies on Close
-// for goroutine termination.
-func (zrd *CompressingReader) Close() error {
-	return zrd.src.Close()
-}
-
-// Apply applies useful options to the lz4 encoder.
-func (zrd *CompressingReader) Apply(options ...Option) (err error) {
-	if zrd.state != crStateInitial {
-		return lz4errors.ErrOptionClosedOrError
-	}
-
-	zrd.Reset(zrd.src)
-
-	for _, o := range options {
-		if err = o(zrd); err != nil {
-			return
-		}
-	}
-	return
-}
-
-func (*CompressingReader) private() {}
-
-func (zrd *CompressingReader) init() error {
-	zrd.frame.InitW(&zrd.out, 1, false)
-	size := zrd.frame.Descriptor.Flags.BlockSizeIndex()
-	zrd.in = size.Get()
-	return zrd.frame.Descriptor.Write(zrd.frame, &zrd.out)
-}
-
-// Read allows reading of lz4 compressed data
-func (zrd *CompressingReader) Read(p []byte) (n int, err error) {
-	defer func() {
-		if err != nil {
-			zrd.state = crStateDone
-		}
-	}()
-
-	if !zrd.out.reset(p) {
-		return len(p), nil
-	}
-
-	switch zrd.state {
-	case crStateInitial:
-		err = zrd.init()
-		if err != nil {
-			return
-		}
-		zrd.state = crStateReading
-	case crStateDone:
-		return 0, errors.New("This reader is done")
-	case crStateFlushing:
-		if zrd.out.dataPos > 0 {
-			n = zrd.out.dataPos
-			zrd.out.data = nil
-			zrd.out.dataPos = 0
-			return
-		} else {
-			zrd.state = crStateDone
-			return 0, io.EOF
-		}
-	}
-
-	for zrd.state == crStateReading {
-		block := zrd.frame.Blocks.Block
-
-		var rCount int
-		rCount, err = io.ReadFull(zrd.src, zrd.in)
-		switch err {
-		case nil:
-			err = block.Compress(
-				zrd.frame, zrd.in[ : rCount], zrd.level,
-			).Write(zrd.frame, &zrd.out)
-			zrd.handler(len(block.Data))
-			if err != nil {
-				return
-			}
-
-			if zrd.out.dataPos == len(zrd.out.data) {
-				n = zrd.out.dataPos
-				zrd.out.dataPos = 0
-				zrd.out.data = nil
-				return
-			}
-		case io.EOF, io.ErrUnexpectedEOF: // read may be partial
-			if rCount > 0 {
-				err = block.Compress(
-					zrd.frame, zrd.in[ : rCount], zrd.level,
-				).Write(zrd.frame, &zrd.out)
-				zrd.handler(len(block.Data))
-				if err != nil {
-					return
-				}
-			}
-
-			err = zrd.frame.CloseW(&zrd.out, 1)
-			if err != nil {
-				return
-			}
-			zrd.state = crStateFlushing
-
-			n = zrd.out.dataPos
-			zrd.out.dataPos = 0
-			zrd.out.data = nil
-			return
-		default:
-			return
-		}
-	}
-
-	err = lz4errors.ErrInternalUnhandledState
-	return
-}
-
-// Reset makes the stream usable again; mostly handy to reuse lz4 encoder
-// instances.
-func (zrd *CompressingReader) Reset(src io.ReadCloser) {
-	zrd.frame.Reset(1)
-	zrd.state = crStateInitial
-	zrd.src = src
-	zrd.out.clear()
-}
-
-type ovWriter struct {
-	data []byte
-	ov []byte
-	dataPos int
-	ovPos int
-}
-
-func (wr *ovWriter) Write(p []byte) (n int, err error) {
-	count := copy(wr.data[wr.dataPos : ], p)
-	wr.dataPos += count
-
-	if count < len(p) {
-		wr.ov = append(wr.ov, p[count : ]...)
-	}
-
-	return len(p), nil
-}
-
-func (wr *ovWriter) reset(out []byte) bool {
-	ovRem := len(wr.ov) - wr.ovPos
-
-	if ovRem >= len(out) {
-		wr.ovPos += copy(out, wr.ov[wr.ovPos : ])
-		return false
-	}
-
-	if ovRem > 0 {
-		copy(out, wr.ov[wr.ovPos : ])
-		wr.ov = wr.ov[ : 0]
-		wr.ovPos = 0
-		wr.dataPos = ovRem
-	} else if wr.ovPos > 0 {
-		wr.ov = wr.ov[ : 0]
-		wr.ovPos = 0
-		wr.dataPos = 0
-	}
-
-	wr.data = out
-	return true
-}
-
-func (wr *ovWriter) clear() {
-	wr.data = nil
-	wr.dataPos = 0
-	wr.ov = wr.ov[ : 0]
-	wr.ovPos = 0
-}

vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go

@@ -8,9 +8,12 @@ const (
 	Block256Kb
 	Block1Mb
 	Block4Mb
-	Block8Mb = 2 * Block4Mb
 )
 
+// In legacy mode all blocks are compressed regardless
+// of the compressed size: use the bound size.
+var Block8Mb = uint32(CompressBlockBound(8 << 20))
+
 var (
 	BlockPool64K  = sync.Pool{New: func() interface{} { return make([]byte, Block64Kb) }}
 	BlockPool256K = sync.Pool{New: func() interface{} { return make([]byte, Block256Kb) }}

vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go

@@ -224,7 +224,9 @@ func (b *FrameDataBlock) Close(f *Frame) {
 func (b *FrameDataBlock) Compress(f *Frame, src []byte, level lz4block.CompressionLevel) *FrameDataBlock {
 	data := b.data
 	if f.isLegacy() {
-		data = data[:cap(data)]
+		// In legacy mode, the buffer is sized according to CompressBlockBound,
+		// but only 8Mb is buffered for compression.
+		src = src[:8<<20]
 	} else {
 		data = data[:len(src)] // trigger the incompressible flag in CompressBlock
 	}

vendor/github.com/pierrec/lz4/v4/options.go

@@ -57,13 +57,6 @@ func BlockSizeOption(size BlockSize) Option {
 			}
 			w.frame.Descriptor.Flags.BlockSizeIndexSet(lz4block.Index(size))
 			return nil
-		case *CompressingReader:
-			size := uint32(size)
-			if !lz4block.IsValid(size) {
-				return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidBlockSize, size)
-			}
-			w.frame.Descriptor.Flags.BlockSizeIndexSet(lz4block.Index(size))
-			return nil
 		}
 		return lz4errors.ErrOptionNotApplicable
 	}
@@ -79,9 +72,6 @@ func BlockChecksumOption(flag bool) Option {
 		case *Writer:
 			w.frame.Descriptor.Flags.BlockChecksumSet(flag)
 			return nil
-		case *CompressingReader:
-			w.frame.Descriptor.Flags.BlockChecksumSet(flag)
-			return nil
 		}
 		return lz4errors.ErrOptionNotApplicable
 	}
@@ -97,9 +87,6 @@ func ChecksumOption(flag bool) Option {
 		case *Writer:
 			w.frame.Descriptor.Flags.ContentChecksumSet(flag)
 			return nil
-		case *CompressingReader:
-			w.frame.Descriptor.Flags.ContentChecksumSet(flag)
-			return nil
 		}
 		return lz4errors.ErrOptionNotApplicable
 	}
@@ -117,10 +104,6 @@ func SizeOption(size uint64) Option {
 			w.frame.Descriptor.Flags.SizeSet(size > 0)
 			w.frame.Descriptor.ContentSize = size
 			return nil
-		case *CompressingReader:
-			w.frame.Descriptor.Flags.SizeSet(size > 0)
-			w.frame.Descriptor.ContentSize = size
-			return nil
 		}
 		return lz4errors.ErrOptionNotApplicable
 	}
@@ -179,14 +162,6 @@ func CompressionLevelOption(level CompressionLevel) Option {
 			}
 			w.level = lz4block.CompressionLevel(level)
 			return nil
-		case *CompressingReader:
-			switch level {
-			case Fast, Level1, Level2, Level3, Level4, Level5, Level6, Level7, Level8, Level9:
-			default:
-				return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidCompressionLevel, level)
-			}
-			w.level = lz4block.CompressionLevel(level)
-			return nil
 		}
 		return lz4errors.ErrOptionNotApplicable
 	}
@@ -211,9 +186,6 @@ func OnBlockDoneOption(handler func(size int)) Option {
 		case *Reader:
 			rw.handler = handler
 			return nil
-		case *CompressingReader:
-			rw.handler = handler
-			return nil
 		}
 		return lz4errors.ErrOptionNotApplicable
 	}

vendor/modules.txt

@@ -26,7 +26,7 @@ github.com/klauspost/compress/internal/cpuinfo
 github.com/klauspost/compress/internal/snapref
 github.com/klauspost/compress/zstd
 github.com/klauspost/compress/zstd/internal/xxhash
-# github.com/pierrec/lz4/v4 v4.1.21
+# github.com/pierrec/lz4/v4 v4.1.19
 ## explicit; go 1.14
 github.com/pierrec/lz4/v4
 github.com/pierrec/lz4/v4/internal/lz4block
@@ -49,7 +49,7 @@ golang.org/x/image/riff
 golang.org/x/image/vp8
 golang.org/x/image/vp8l
 golang.org/x/image/webp
-# golang.org/x/net v0.20.0
+# golang.org/x/net v0.19.0
 ## explicit; go 1.18
 golang.org/x/net/html
 golang.org/x/net/html/atom