diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 69ababbd1..2e3d49aa8 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -1103,7 +1103,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str PartNumber: part.PartNumber, } } - checksumCombined = append(checksumCombined, cs.Raw()...) + checksumCombined = append(checksumCombined, cs.Raw...) } // All parts except the last part has to be at least 5MB. diff --git a/cmd/erasure-single-drive.go b/cmd/erasure-single-drive.go index 9f525dcf5..89b23f152 100644 --- a/cmd/erasure-single-drive.go +++ b/cmd/erasure-single-drive.go @@ -1017,6 +1017,11 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st } fi.DataDir = mustGetUUID() + fi.Checksum = opts.WantChecksum.AppendTo(nil) + if opts.EncryptFn != nil { + fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) + } + uniqueID := mustGetUUID() tempObj := uniqueID @@ -2593,6 +2598,7 @@ func (es *erasureSingle) ListObjectParts(ctx context.Context, bucket, object, up result.MaxParts = maxParts result.PartNumberMarker = partNumberMarker result.UserDefined = cloneMSS(fi.Metadata) + result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum] // For empty number of parts or maxParts as zero, return right here. if len(fi.Parts) == 0 || maxParts == 0 { @@ -2613,10 +2619,15 @@ func (es *erasureSingle) ListObjectParts(ctx context.Context, bucket, object, up count := maxParts for _, part := range parts { result.Parts = append(result.Parts, PartInfo{ - PartNumber: part.Number, - ETag: part.ETag, - LastModified: fi.ModTime, - Size: part.Size, + PartNumber: part.Number, + ETag: part.ETag, + LastModified: fi.ModTime, + ActualSize: part.ActualSize, + Size: part.Size, + ChecksumCRC32: part.Checksums["CRC32"], + ChecksumCRC32C: part.Checksums["CRC32C"], + ChecksumSHA1: part.Checksums["SHA1"], + ChecksumSHA256: part.Checksums["SHA256"], }) count-- if count == 0 { @@ -2685,6 +2696,21 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str return oi, err } + // Checksum type set when upload started. + var checksumType hash.ChecksumType + if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" { + checksumType = hash.NewChecksumType(cs) + if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) { + return oi, InvalidArgument{ + Bucket: bucket, + Object: fi.Name, + Err: fmt.Errorf("checksum type mismatch"), + } + } + } + + var checksumCombined []byte + // However, in case of encryption, the persisted part ETags don't match // what we have sent to the client during PutObjectPart. The reason is // that ETags are encrypted. Hence, the client will send a list of complete @@ -2731,6 +2757,7 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str } return oi, invp } + expPart := currentFI.Parts[partIdx] // ensure that part ETag is canonicalized to strip off extraneous quotes part.ETag = canonicalizeETag(part.ETag) @@ -2744,27 +2771,58 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str return oi, invp } + if checksumType.IsSet() { + crc := expPart.Checksums[checksumType.String()] + if crc == "" { + return oi, InvalidPart{ + PartNumber: part.PartNumber, + } + } + wantCS := map[string]string{ + hash.ChecksumCRC32.String(): part.ChecksumCRC32, + hash.ChecksumCRC32C.String(): part.ChecksumCRC32C, + hash.ChecksumSHA1.String(): part.ChecksumSHA1, + hash.ChecksumSHA256.String(): part.ChecksumSHA256, + } + if wantCS[checksumType.String()] != crc { + return oi, InvalidPart{ + PartNumber: part.PartNumber, + ExpETag: wantCS[checksumType.String()], + GotETag: crc, + } + } + cs := hash.NewChecksumString(checksumType.String(), crc) + if !cs.Valid() { + return oi, InvalidPart{ + PartNumber: part.PartNumber, + } + } + checksumCombined = append(checksumCombined, cs.Raw...) + } + // All parts except the last part has to be atleast 5MB. if (i < len(parts)-1) && !isMinAllowedPartSize(currentFI.Parts[partIdx].ActualSize) { return oi, PartTooSmall{ PartNumber: part.PartNumber, - PartSize: currentFI.Parts[partIdx].ActualSize, + PartSize: expPart.ActualSize, PartETag: part.ETag, } } // Save for total object size. - objectSize += currentFI.Parts[partIdx].Size + objectSize += expPart.Size // Save the consolidated actual size. - objectActualSize += currentFI.Parts[partIdx].ActualSize + objectActualSize += expPart.ActualSize // Add incoming parts. fi.Parts[i] = ObjectPartInfo{ Number: part.PartNumber, - Size: currentFI.Parts[partIdx].Size, - ActualSize: currentFI.Parts[partIdx].ActualSize, - Index: currentFI.Parts[partIdx].Index, + Size: expPart.Size, + ActualSize: expPart.ActualSize, + ModTime: expPart.ModTime, + Index: expPart.Index, + Checksums: nil, // Not transferred since we do not need it. } } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 5957e3c9d..99adf5689 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1797,7 +1797,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } - if err = actualReader.AddChecksum(r, false); err != nil { + if err = actualReader.AddChecksum(r, globalIsGateway); err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } @@ -1818,7 +1818,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } - if err := hashReader.AddChecksum(r, size < 0); err != nil { + if err := hashReader.AddChecksum(r, size < 0 || globalIsGateway); err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } @@ -2129,7 +2129,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } - if err = hreader.AddChecksum(r, false); err != nil { + if err = hreader.AddChecksum(r, globalIsGateway); err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index caae55040..d215c4255 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -390,7 +390,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } - if err = actualReader.AddChecksum(r, false); err != nil { + if err = actualReader.AddChecksum(r, globalIsGateway); err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } @@ -411,7 +411,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } - if err := hashReader.AddChecksum(r, size < 0); err != nil { + if err := hashReader.AddChecksum(r, size < 0 || globalIsGateway); err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } diff --git a/internal/hash/checksum.go b/internal/hash/checksum.go index 0efdae62a..ce2a09591 100644 --- a/internal/hash/checksum.go +++ b/internal/hash/checksum.go @@ -18,9 +18,11 @@ package hash import ( + "bytes" "crypto/sha1" "encoding/base64" "encoding/binary" + "fmt" "hash" "hash/crc32" "net/http" @@ -61,6 +63,7 @@ const ( type Checksum struct { Type ChecksumType Encoded string + Raw []byte } // Is returns if c is all of t. @@ -169,7 +172,8 @@ func NewChecksumFromData(t ChecksumType, data []byte) *Checksum { } h := t.Hasher() h.Write(data) - c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(h.Sum(nil))} + raw := h.Sum(nil) + c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(raw), Raw: raw} if !c.Valid() { return nil } @@ -200,19 +204,27 @@ func ReadCheckSums(b []byte) map[string]string { return res } -// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value. -func NewChecksumString(alg, value string) *Checksum { - t := NewChecksumType(alg) - if !t.IsSet() { +// NewChecksumWithType is similar to NewChecksumString but expects input algo of ChecksumType. +func NewChecksumWithType(alg ChecksumType, value string) *Checksum { + if !alg.IsSet() { return nil } - c := Checksum{Type: t, Encoded: value} + bvalue, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil + } + c := Checksum{Type: alg, Encoded: value, Raw: bvalue} if !c.Valid() { return nil } return &c } +// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value. +func NewChecksumString(alg, value string) *Checksum { + return NewChecksumWithType(NewChecksumType(alg), value) +} + // AppendTo will append the checksum to b. // ReadCheckSums reads the values back. func (c *Checksum) AppendTo(b []byte) []byte { @@ -221,7 +233,7 @@ func (c *Checksum) AppendTo(b []byte) []byte { } var tmp [binary.MaxVarintLen32]byte n := binary.PutUvarint(tmp[:], uint64(c.Type)) - crc := c.Raw() + crc := c.Raw if len(crc) != c.Type.RawByteLen() { return b } @@ -238,19 +250,10 @@ func (c Checksum) Valid() bool { if len(c.Encoded) == 0 || c.Type.Is(ChecksumTrailing) { return c.Type.Is(ChecksumNone) || c.Type.Is(ChecksumTrailing) } - raw := c.Raw() + raw := c.Raw return c.Type.RawByteLen() == len(raw) } -// Raw returns the Raw checksum. -func (c Checksum) Raw() []byte { - if len(c.Encoded) == 0 { - return nil - } - v, _ := base64.StdEncoding.DecodeString(c.Encoded) - return v -} - // Matches returns whether given content matches c. func (c Checksum) Matches(content []byte) error { if len(c.Encoded) == 0 { @@ -261,11 +264,11 @@ func (c Checksum) Matches(content []byte) error { if err != nil { return err } - got := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - if got != c.Encoded { + sum := hasher.Sum(nil) + if !bytes.Equal(sum, c.Raw) { return ChecksumMismatch{ Want: c.Encoded, - Got: got, + Got: base64.StdEncoding.EncodeToString(sum), } } return nil @@ -292,13 +295,13 @@ func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) { // AddChecksumHeader will transfer any checksum value that has been checked. func AddChecksumHeader(w http.ResponseWriter, c map[string]string) { for k, v := range c { - typ := NewChecksumType(k) - if !typ.IsSet() { + fmt.Println(c, v) + cksum := NewChecksumString(k, v) + if cksum == nil { continue } - crc := Checksum{Type: typ, Encoded: v} - if crc.Valid() { - w.Header().Set(typ.Key(), v) + if cksum.Valid() { + w.Header().Set(cksum.Type.Key(), v) } } } @@ -314,12 +317,11 @@ func GetContentChecksum(r *http.Request) (*Checksum, error) { } return nil, ErrInvalidChecksum } - c := Checksum{Type: t, Encoded: s} - if !c.Valid() { + cksum := NewChecksumWithType(t, s) + if cksum == nil { return nil, ErrInvalidChecksum } - - return &c, nil + return cksum, nil } // getContentChecksum returns content checksum type and value. diff --git a/internal/hash/reader.go b/internal/hash/reader.go index 7b2b7b5e1..a155a2fcb 100644 --- a/internal/hash/reader.go +++ b/internal/hash/reader.go @@ -186,7 +186,7 @@ func (r *Reader) Read(p []byte) (int, error) { } } if r.contentHasher != nil { - if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw(), sum) { + if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw, sum) { err := ChecksumMismatch{ Want: r.contentHash.Encoded, Got: base64.StdEncoding.EncodeToString(sum),