diff --git a/Makefile b/Makefile index feadc3251..cea8ad39b 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ checks: getdeps: @mkdir -p ${GOPATH}/bin @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) - @which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3) - @which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer) + @which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3) + @which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer) crosscompile: @(env bash $(PWD)/buildscripts/cross-compile.sh) diff --git a/cmd/metacache-walk.go b/cmd/metacache-walk.go index c8d5d3f0a..056cc40f8 100644 --- a/cmd/metacache-walk.go +++ b/cmd/metacache-walk.go @@ -88,7 +88,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // Fast exit track to check if we are listing an object with // a trailing slash, this will avoid to list the object content. if HasSuffix(opts.BaseDir, SlashSeparator) { - metadata, err := xioutil.ReadFile(pathJoin(volumeDir, + metadata, err := s.readMetadata(pathJoin(volumeDir, opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix, xlStorageFormatFile)) if err == nil { @@ -97,7 +97,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // behavior. out <- metaCacheEntry{ name: opts.BaseDir, - metadata: xlMetaV2TrimData(metadata), + metadata: metadata, } } else { if st, err := os.Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() { @@ -154,25 +154,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // If root was an object return it as such. if HasSuffix(entry, xlStorageFormatFile) { var meta metaCacheEntry - f, err := os.OpenFile(pathJoin(volumeDir, current, entry), readMode, 0) + meta.metadata, err = s.readMetadata(pathJoin(volumeDir, current, entry)) if err != nil { logger.LogIf(ctx, err) continue } - stat, err := f.Stat() - if err != nil { - logger.LogIf(ctx, err) - f.Close() - continue - } - meta.metadata, err = readXLMetaNoData(f, stat.Size()) - if err != nil { - logger.LogIf(ctx, err) - f.Close() - continue - } - f.Close() - meta.metadata = xlMetaV2TrimData(meta.metadata) meta.name = strings.TrimSuffix(entry, xlStorageFormatFile) meta.name = strings.TrimSuffix(meta.name, SlashSeparator) meta.name = pathJoin(current, meta.name) @@ -233,7 +219,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash } - meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile)) + meta.metadata, err = s.readMetadata(pathJoin(volumeDir, meta.name, xlStorageFormatFile)) switch { case err == nil: // It was an object diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index cf18ba1a7..533c392c9 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -1420,7 +1420,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) { err = readMore(size) return buf, err case 1, 2: - sz, tmp, err := ReadBytesHeader(tmp) + sz, tmp, err := msgp.ReadBytesHeader(tmp) if err != nil { return nil, err } @@ -1459,51 +1459,3 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) { return nil, errors.New("unknown major metadata version") } } - -// ReadBytesHeader reads the 'bin' header size -// off of 'b' and returns the size and remaining bytes. -// Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a bin object) -// TODO: Replace when https://github.com/tinylib/msgp/pull/289 is merged. -func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) { - if len(b) < 1 { - return 0, nil, msgp.ErrShortBytes - } - var big = binary.BigEndian - - const ( - mbin8 uint8 = 0xc4 - mbin16 uint8 = 0xc5 - mbin32 uint8 = 0xc6 - ) - switch b[0] { - case mbin8: - if len(b) < 2 { - err = msgp.ErrShortBytes - return - } - sz = uint32(b[1]) - o = b[2:] - return - case mbin16: - if len(b) < 3 { - err = msgp.ErrShortBytes - return - } - sz = uint32(big.Uint16(b[1:])) - o = b[3:] - return - case mbin32: - if len(b) < 5 { - err = msgp.ErrShortBytes - return - } - sz = big.Uint32(b[1:]) - o = b[5:] - return - default: - err = msgp.TypeError{Method: msgp.BinType, Encoded: msgp.NextType(b)} - return - } -} diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index f23ece729..eb2873911 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -384,6 +384,19 @@ func (s *xlStorage) Healing() *healingTracker { return &h } +func (s *xlStorage) readMetadata(itemPath string) ([]byte, error) { + f, err := os.OpenFile(itemPath, readMode, 0) + if err != nil { + return nil, err + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return nil, err + } + return readXLMetaNoData(f, stat.Size()) +} + func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) { // Updates must be closed before we return. defer close(updates) @@ -421,29 +434,13 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates return sizeSummary{}, errSkipFile } - f, err := os.OpenFile(item.Path, readMode, 0) + buf, err := s.readMetadata(item.Path) if err != nil { if intDataUpdateTracker.debug { console.Debugf(color.Green("scannerBucket:")+" object path missing: %v: %w\n", item.Path, err) } return sizeSummary{}, errSkipFile } - defer f.Close() - stat, err := f.Stat() - if err != nil { - if intDataUpdateTracker.debug { - console.Debugf(color.Green("scannerBucket:")+" stat failed: %v: %w\n", item.Path, err) - } - return sizeSummary{}, errSkipFile - } - - buf, err := readXLMetaNoData(f, stat.Size()) - if err != nil { - if intDataUpdateTracker.debug { - console.Debugf(color.Green("scannerBucket:")+" readXLMetaNoData: %v: %w\n", item.Path, err) - } - return sizeSummary{}, errSkipFile - } // Remove filename which is the meta file. item.transformMetaDir() diff --git a/go.mod b/go.mod index 4e2b40263..55768eaee 100644 --- a/go.mod +++ b/go.mod @@ -77,8 +77,7 @@ require ( github.com/shirou/gopsutil/v3 v3.21.3 github.com/spf13/pflag v1.0.5 // indirect github.com/streadway/amqp v1.0.0 - github.com/tinylib/msgp v1.1.3 - github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect + github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 github.com/valyala/bytebufferpool v1.0.0 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a github.com/willf/bloom v2.0.3+incompatible diff --git a/go.sum b/go.sum index 234207658..1c8fd6257 100644 --- a/go.sum +++ b/go.sum @@ -686,16 +686,15 @@ github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.6 h1:8fDdlahON04OZBlTQCIatW8FstSFJz8oxidj5h0rmSQ= github.com/tidwall/sjson v1.1.6/go.mod h1:KN3FZ7odvXIHPbJdhNorK/M9lWweVUbXsXXhrJ/kGOA= -github.com/tinylib/msgp v1.1.3 h1:3giwAkmtaEDLSV0MdO1lDLuPgklgPzmk8H9+So2BVfA= github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 h1:x5VlSgDgIGXNegkO4gigpYmb/RFkKGgy12Kkrbif7XE= +github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -989,6 +988,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=