Introduce simpler GetMultipartInfo call for performance (#9722)

Advantages avoids 100's of stats which are needed for each
upload operation in FS/NAS gateway mode when uploading a large
multipart object, dramatically increases performance for
multipart uploads by avoiding recursive calls.

For other gateway's simplifies the approach since
azure, gcs, hdfs gateway's don't capture any specific
metadata during upload which needs handler validation
for encryption/compression.

Erasure coding was already optimized, additionally
just avoids small allocations of large data structure.

Fixes #7206
This commit is contained in:
Harshavardhana
2020-05-28 12:36:20 -07:00
committed by GitHub
parent 231c5cf6de
commit b330c2c57e
14 changed files with 292 additions and 71 deletions

View File

@@ -194,6 +194,7 @@ type ObjectInfo struct {
PutObjReader *PutObjReader `json:"-"`
metadataOnly bool
keyRotation bool
// Date and time when the object was last accessed.
AccTime time.Time
@@ -202,6 +203,28 @@ type ObjectInfo struct {
backendType BackendType
}
// MultipartInfo captures metadata information about the uploadId
// this data structure is used primarily for some internal purposes
// for verifying upload type such as was the upload
// - encrypted
// - compressed
type MultipartInfo struct {
// Name of the bucket.
Bucket string
// Name of the object.
Object string
// Upload ID identifying the multipart upload whose parts are being listed.
UploadID string
// Date and time at which the multipart upload was initiated.
Initiated time.Time
// Any metadata set during InitMultipartUpload, including encryption headers.
UserDefined map[string]string
}
// ListPartsInfo - represents list of all parts.
type ListPartsInfo struct {
// Name of the bucket.
@@ -235,8 +258,6 @@ type ListPartsInfo struct {
// Any metadata set during InitMultipartUpload, including encryption headers.
UserDefined map[string]string
EncodingType string // Not supported yet.
}
// Lookup - returns if uploadID is valid
@@ -362,20 +383,6 @@ type PartInfo struct {
ActualSize int64
}
// MultipartInfo - represents metadata in progress multipart upload.
type MultipartInfo struct {
// Object name for which the multipart upload was initiated.
Object string
// Unique identifier for this multipart upload.
UploadID string
// Date and time at which the multipart upload was initiated.
Initiated time.Time
StorageClass string // Not supported yet.
}
// CompletePart - represents the part that was completed, this is sent by the client
// during CompleteMultipartUpload request.
type CompletePart struct {