From 1f659204a275630ba68c5b3193a65ee1f603266f Mon Sep 17 00:00:00 2001 From: Andreas Auernhammer Date: Fri, 26 Feb 2021 18:52:02 +0100 Subject: [PATCH] remove GetObject from ObjectLayer interface (#11635) This commit removes the `GetObject` method from the `ObjectLayer` interface. The `GetObject` method is not longer used by the HTTP handlers implementing the high-level S3 semantics. Instead, they use the `GetObjectNInfo` method which returns both, an object handle as well as the object metadata. Therefore, it is no longer necessary that a concrete `ObjectLayer` implements `GetObject`. --- cmd/benchmark-utils_test.go | 138 ------- cmd/erasure-server-pool.go | 21 - cmd/erasure-sets.go | 8 - cmd/fs-v1.go | 35 -- cmd/gateway/azure/gateway-azure.go | 4 +- cmd/gateway/gcs/gateway-gcs.go | 4 +- cmd/gateway/hdfs/gateway-hdfs.go | 4 +- cmd/gateway/s3/gateway-s3-sse.go | 6 +- cmd/gateway/s3/gateway-s3.go | 4 +- cmd/object-api-getobject_test.go | 613 ----------------------------- cmd/object-api-interface.go | 21 +- cmd/object_api_suite_test.go | 10 +- cmd/web-handlers_test.go | 2 +- 13 files changed, 37 insertions(+), 833 deletions(-) delete mode 100644 cmd/object-api-getobject_test.go diff --git a/cmd/benchmark-utils_test.go b/cmd/benchmark-utils_test.go index 081448468..fe3108142 100644 --- a/cmd/benchmark-utils_test.go +++ b/cmd/benchmark-utils_test.go @@ -19,7 +19,6 @@ package cmd import ( "bytes" "context" - "io/ioutil" "math" "math/rand" "strconv" @@ -175,56 +174,6 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) runPutObjectBenchmarkParallel(b, objLayer, objSize) } -// Benchmark utility functions for ObjectLayer.GetObject(). -// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark. -func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { - // obtains random bucket name. - bucket := getRandomBucketName() - // create bucket. - err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{}) - if err != nil { - b.Fatal(err) - } - - textData := generateBytesData(objSize) - - // generate etag for the generated data. - // etag of the data to written is required as input for PutObject. - // PutObject is the functions which writes the data onto the FS/Erasure backend. - - // get text data generated for number of bytes equal to object size. - md5hex := getMD5Hash(textData) - sha256hex := "" - - for i := 0; i < 10; i++ { - // insert the object. - var objInfo ObjectInfo - objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i), - mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{}) - if err != nil { - b.Fatal(err) - } - if objInfo.ETag != md5hex { - b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex) - } - } - - // benchmark utility which helps obtain number of allocations and bytes allocated per ops. - b.ReportAllocs() - // the actual benchmark for GetObject starts here. Reset the benchmark timer. - b.ResetTimer() - for i := 0; i < b.N; i++ { - var buffer = new(bytes.Buffer) - err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "", ObjectOptions{}) - if err != nil { - b.Error(err) - } - } - // Benchmark ends here. Stop timer. - b.StopTimer() - -} - // randomly picks a character and returns its equivalent byte array. func getRandomByte() []byte { const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -240,38 +189,6 @@ func generateBytesData(size int) []byte { return bytes.Repeat(getRandomByte(), size) } -// creates Erasure/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function. -func benchmarkGetObject(b *testing.B, instanceType string, objSize int) { - // create a temp Erasure/FS backend. - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - objLayer, disks, err := prepareTestBackend(ctx, instanceType) - if err != nil { - b.Fatalf("Failed obtaining Temp Backend: %s", err) - } - // cleaning up the backend by removing all the directories and files created. - defer removeRoots(disks) - - // uses *testing.B and the object Layer to run the benchmark. - runGetObjectBenchmark(b, objLayer, objSize) -} - -// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() . -func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) { - // create a temp Erasure/FS backend. - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - objLayer, disks, err := prepareTestBackend(ctx, instanceType) - if err != nil { - b.Fatalf("Failed obtaining Temp Backend: %s", err) - } - // cleaning up the backend by removing all the directories and files created. - defer removeRoots(disks) - - // uses *testing.B and the object Layer to run the benchmark. - runGetObjectBenchmarkParallel(b, objLayer, objSize) -} - // Parallel benchmark utility functions for ObjectLayer.PutObject(). // Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark. func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { @@ -315,58 +232,3 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { // Benchmark ends here. Stop timer. b.StopTimer() } - -// Parallel benchmark utility functions for ObjectLayer.GetObject(). -// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark. -func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { - // obtains random bucket name. - bucket := getRandomBucketName() - // create bucket. - err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{}) - if err != nil { - b.Fatal(err) - } - - // get text data generated for number of bytes equal to object size. - textData := generateBytesData(objSize) - // generate md5sum for the generated data. - // md5sum of the data to written is required as input for PutObject. - // PutObject is the functions which writes the data onto the FS/Erasure backend. - - md5hex := getMD5Hash([]byte(textData)) - sha256hex := "" - - for i := 0; i < 10; i++ { - // insert the object. - var objInfo ObjectInfo - objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i), - mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{}) - if err != nil { - b.Fatal(err) - } - if objInfo.ETag != md5hex { - b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex) - } - } - - // benchmark utility which helps obtain number of allocations and bytes allocated per ops. - b.ReportAllocs() - // the actual benchmark for GetObject starts here. Reset the benchmark timer. - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - i := 0 - for pb.Next() { - err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "", ObjectOptions{}) - if err != nil { - b.Error(err) - } - i++ - if i == 10 { - i = 0 - } - } - }) - // Benchmark ends here. Stop timer. - b.StopTimer() - -} diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 627702154..580e62f40 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -595,27 +595,6 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object return gr, ObjectNotFound{Bucket: bucket, Object: object} } -func (z *erasureServerPools) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error { - if err := checkGetObjArgs(ctx, bucket, object); err != nil { - return err - } - - object = encodeDirObject(object) - for _, pool := range z.serverPools { - if err := pool.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil { - if isErrObjectNotFound(err) || isErrVersionNotFound(err) { - continue - } - return err - } - return nil - } - if opts.VersionID != "" { - return VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID} - } - return ObjectNotFound{Bucket: bucket, Object: object} -} - func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { if err = checkGetObjArgs(ctx, bucket, object); err != nil { return objInfo, err diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index ef81e0ab6..21fc37957 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "hash/crc32" - "io" "math/rand" "net/http" "sort" @@ -784,13 +783,6 @@ func (s *erasureSets) GetObjectNInfo(ctx context.Context, bucket, object string, return set.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts) } -// GetObject - reads an object from the hashedSet based on the object name. -func (s *erasureSets) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error { - set := s.getHashedSet(object) - auditObjectErasureSet(ctx, object, set, s.poolNumber) - return set.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts) -} - func (s *erasureSets) parentDirIsObject(ctx context.Context, bucket, parent string) bool { if parent == "." { return false diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 37112130d..8423f2ac4 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -768,41 +768,6 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string, return objReaderFn(reader, h, opts.CheckPrecondFn, closeFn) } -// GetObject - reads an object from the disk. -// Supports additional parameters like offset and length -// which are synonymous with HTTP Range requests. -// -// startOffset indicates the starting read location of the object. -// length indicates the total length of the object. -func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) { - if opts.VersionID != "" && opts.VersionID != nullVersionID { - return VersionNotFound{ - Bucket: bucket, - Object: object, - VersionID: opts.VersionID, - } - } - - if err = checkGetObjArgs(ctx, bucket, object); err != nil { - return err - } - - // Lock the object before reading. - lk := fs.NewNSLock(bucket, object) - if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil { - logger.LogIf(ctx, err) - return err - } - defer lk.RUnlock() - - atomic.AddInt64(&fs.activeIOCount, 1) - defer func() { - atomic.AddInt64(&fs.activeIOCount, -1) - }() - - return fs.getObject(ctx, bucket, object, offset, length, writer, etag, true) -} - // getObject - wrapper for GetObject func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string, lock bool) (err error) { if _, err = fs.statBucketDir(ctx, bucket); err != nil { diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index a18c4606e..2bb56511b 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -799,7 +799,7 @@ func (a *azureObjects) GetObjectNInfo(ctx context.Context, bucket, object string pr, pw := io.Pipe() go func() { - err := a.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.InnerETag, opts) + err := a.getObject(ctx, bucket, object, startOffset, length, pw, objInfo.InnerETag, opts) pw.CloseWithError(err) }() // Setup cleanup function to cause the above go-routine to @@ -814,7 +814,7 @@ func (a *azureObjects) GetObjectNInfo(ctx context.Context, bucket, object string // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { +func (a *azureObjects) getObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { // startOffset cannot be negative. if startOffset < 0 { return azureToObjectError(minio.InvalidRange{}, bucket, object) diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 20813aaf4..573078a11 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -751,7 +751,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string, pr, pw := io.Pipe() go func() { - err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts) + err := l.getObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts) pw.CloseWithError(err) }() // Setup cleanup function to cause the above go-routine to @@ -766,7 +766,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string, // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { +func (l *gcsGateway) getObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil { diff --git a/cmd/gateway/hdfs/gateway-hdfs.go b/cmd/gateway/hdfs/gateway-hdfs.go index 10c3bef20..1419c89df 100644 --- a/cmd/gateway/hdfs/gateway-hdfs.go +++ b/cmd/gateway/hdfs/gateway-hdfs.go @@ -591,7 +591,7 @@ func (n *hdfsObjects) GetObjectNInfo(ctx context.Context, bucket, object string, pr, pw := io.Pipe() go func() { - nerr := n.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts) + nerr := n.getObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts) pw.CloseWithError(nerr) }() @@ -614,7 +614,7 @@ func (n *hdfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstB }) } -func (n *hdfsObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { +func (n *hdfsObjects) getObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { if _, err := n.clnt.Stat(n.hdfsPathJoin(bucket)); err != nil { return hdfsToObjectErr(ctx, err, bucket) } diff --git a/cmd/gateway/s3/gateway-s3-sse.go b/cmd/gateway/s3/gateway-s3-sse.go index b46f01db4..66836b781 100644 --- a/cmd/gateway/s3/gateway-s3-sse.go +++ b/cmd/gateway/s3/gateway-s3-sse.go @@ -226,7 +226,7 @@ func (l *s3EncObjects) getGWMetadata(ctx context.Context, bucket, metaFileName s return m, err1 } var buffer bytes.Buffer - err = l.s3Objects.GetObject(ctx, bucket, metaFileName, 0, oi.Size, &buffer, oi.ETag, minio.ObjectOptions{}) + err = l.s3Objects.getObject(ctx, bucket, metaFileName, 0, oi.Size, &buffer, oi.ETag, minio.ObjectOptions{}) if err != nil { return m, err } @@ -272,7 +272,7 @@ func (l *s3EncObjects) getObject(ctx context.Context, bucket string, key string, dmeta, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(key)) if err != nil { // unencrypted content - return l.s3Objects.GetObject(ctx, bucket, key, startOffset, length, writer, etag, o) + return l.s3Objects.getObject(ctx, bucket, key, startOffset, length, writer, etag, o) } if startOffset < 0 { logger.LogIf(ctx, minio.InvalidRange{}) @@ -303,7 +303,7 @@ func (l *s3EncObjects) getObject(ctx context.Context, bucket string, key string, if _, _, err := dmeta.ObjectToPartOffset(ctx, endOffset); err != nil { return minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size} } - return l.s3Objects.GetObject(ctx, bucket, key, partOffset, endOffset, writer, dmeta.ETag, o) + return l.s3Objects.getObject(ctx, bucket, key, partOffset, endOffset, writer, dmeta.ETag, o) } // GetObjectNInfo - returns object info and locked object ReadCloser diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index d275821e9..72eacd73e 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -403,7 +403,7 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r pr, pw := io.Pipe() go func() { - err := l.GetObject(ctx, bucket, object, off, length, pw, objInfo.ETag, opts) + err := l.getObject(ctx, bucket, object, off, length, pw, objInfo.ETag, opts) pw.CloseWithError(err) }() @@ -419,7 +419,7 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, o minio.ObjectOptions) error { +func (l *s3Objects) getObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, o minio.ObjectOptions) error { if length < 0 && length != -1 { return minio.ErrorRespToObjectError(minio.InvalidRange{}, bucket, key) } diff --git a/cmd/object-api-getobject_test.go b/cmd/object-api-getobject_test.go deleted file mode 100644 index 645f2eb10..000000000 --- a/cmd/object-api-getobject_test.go +++ /dev/null @@ -1,613 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2016, 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "runtime" - "strings" - "testing" - - humanize "github.com/dustin/go-humanize" -) - -// Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup. -func TestGetObject(t *testing.T) { - ExecExtendedObjectLayerTest(t, testGetObject) -} - -// ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated. -func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) { - // Setup for the tests. - bucketName := getRandomBucketName() - objectName := "test-object" - emptyDirName := "test-empty-dir/" - - // create bucket. - err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{}) - // Stop the test if creation of the bucket fails. - if err != nil { - t.Fatalf("%s : %s", instanceType, err.Error()) - } - - // set of byte data for PutObject. - // object has to be created before running tests for GetObject. - // this is required even to assert the GetObject data, - // since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical. - bytesData := []struct { - byteData []byte - }{ - // Regular data - {generateBytesData(6 * humanize.MiByte)}, - // Empty data for empty directory - {}, - } - // set of inputs for uploading the objects before tests for downloading is done. - putObjectInputs := []struct { - bucketName string - objectName string - contentLength int64 - textData []byte - metaData map[string]string - }{ - // case - 1. - {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, - {bucketName, emptyDirName, int64(len(bytesData[1].byteData)), bytesData[1].byteData, make(map[string]string)}, - } - // iterate through the above set of inputs and upkoad the object. - for i, input := range putObjectInputs { - // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), ObjectOptions{UserDefined: input.metaData}) - // if object upload fails stop the test. - if err != nil { - t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) - } - } - // set of empty buffers used to fill GetObject data. - buffers := []*bytes.Buffer{ - new(bytes.Buffer), - new(bytes.Buffer), - new(bytes.Buffer), - } - - // test cases with set of inputs - testCases := []struct { - bucketName string - objectName string - startOffset int64 - length int64 - // data obtained/fetched from GetObject. - getObjectData *bytes.Buffer - // writer which governs the write into the `getObjectData`. - writer io.Writer - // flag indicating whether the test for given ase should pass. - shouldPass bool - // expected Result. - expectedData []byte - err error - }{ - // Test case 1-4. - // Cases with invalid bucket names. - {".test", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: .test")}, - {"------", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: ------")}, - {"$this-is-not-valid-too", "obj", 0, 0, nil, nil, false, - []byte(""), fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")}, - {"a", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: a")}, - // Test case - 5. - // Case with invalid object names. - {bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"/")}, - // Test case - 6. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF}, - // Test case with start offset set to 0 and length set to size of the object. - // Fetching the entire object. - // Test case - 7. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], true, bytesData[0].byteData, nil}, - // Test case with `length` parameter set to a negative value. - // Test case - 8. - {bucketName, objectName, 0, int64(-1), buffers[1], buffers[1], true, bytesData[0].byteData, nil}, - // Test case with content-range 1 to objectSize . - // Test case - 9. - {bucketName, objectName, 1, int64(len(bytesData[0].byteData) - 1), buffers[1], buffers[1], true, bytesData[0].byteData[1:], nil}, - // Test case with content-range 100 to objectSize - 100. - // Test case - 10. - {bucketName, objectName, 100, int64(len(bytesData[0].byteData) - 200), buffers[1], buffers[1], true, - bytesData[0].byteData[100 : len(bytesData[0].byteData)-100], nil}, - // Test case with offset greater than the size of the object - // Test case - 11. - {bucketName, objectName, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), buffers[0], - NewEOFWriter(buffers[0], 100), false, []byte{}, - InvalidRange{int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}}, - // Test case with offset greater than the size of the object. - // Test case - 12. - {bucketName, objectName, -1, int64(len(bytesData[0].byteData)), buffers[0], new(bytes.Buffer), false, []byte{}, errUnexpected}, - // Test case length parameter is more than the object size. - // Test case - 13. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData) + 1), buffers[1], buffers[1], false, bytesData[0].byteData, - InvalidRange{0, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData))}}, - // Test case with offset + length > objectSize parameter set to a negative value. - // Test case - 14. - {bucketName, objectName, 2, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], false, bytesData[0].byteData, - InvalidRange{2, int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}}, - // Test case with the writer set to nil. - // Test case - 15. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], nil, false, bytesData[0].byteData, errUnexpected}, - // Test case - 16. - // Test case when it is an empty directory - {bucketName, emptyDirName, 0, int64(len(bytesData[1].byteData)), buffers[2], buffers[2], true, bytesData[1].byteData, nil}, - } - - for i, testCase := range testCases { - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{}) - if err != nil && testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) - } - if err == nil && !testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead.", i+1, instanceType, testCase.err.Error()) - } - // Failed as expected, but does it fail for the expected reason. - if err != nil && !testCase.shouldPass { - if !strings.Contains(err.Error(), testCase.err.Error()) { - t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.err.Error(), err.Error()) - } - } - // Since there are cases for which GetObject fails, this is - // necessary. Test passes as expected, but the output values - // are verified for correctness here. - if err == nil && testCase.shouldPass { - if !bytes.Equal(testCase.expectedData, testCase.getObjectData.Bytes()) { - t.Errorf("Test %d: %s: Data Mismatch: Expected data and the fetched data from GetObject doesn't match.", i+1, instanceType) - } - // empty the buffer so that it can be used to further cases. - testCase.getObjectData.Reset() - } - } -} - -// Wrapper for calling GetObject with permission denied expected -func TestGetObjectPermissionDenied(t *testing.T) { - // Windows doesn't support Chmod under golang - if runtime.GOOS != globalWindowsOSName { - ExecObjectLayerDiskAlteredTest(t, testGetObjectPermissionDenied) - } -} - -// Test GetObject when we are allowed to access some dirs and objects -func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks []string, t *testing.T) { - // Setup for the tests. - bucketName := getRandomBucketName() - // create bucket. - err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{}) - // Stop the test if creation of the bucket fails. - if err != nil { - t.Fatalf("%s : %s", instanceType, err.Error()) - } - - bytesData := []struct { - byteData []byte - }{ - {generateBytesData(6 * humanize.MiByte)}, - } - // set of inputs for uploading the objects before tests for downloading is done. - putObjectInputs := []struct { - bucketName string - objectName string - contentLength int64 - textData []byte - metaData map[string]string - }{ - {bucketName, "test-object1", int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, - {bucketName, "test-object2", int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, - {bucketName, "dir/test-object3", int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, - } - // iterate through the above set of inputs and upkoad the object. - for i, input := range putObjectInputs { - // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), ObjectOptions{UserDefined: input.metaData}) - // if object upload fails stop the test. - if err != nil { - t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) - } - } - - // set of empty buffers used to fill GetObject data. - buffers := []*bytes.Buffer{ - new(bytes.Buffer), - } - - // test cases with set of inputs - testCases := []struct { - bucketName string - objectName string - chmodPath string - startOffset int64 - length int64 - // data obtained/fetched from GetObject. - getObjectData *bytes.Buffer - // writer which governs the write into the `getObjectData`. - writer io.Writer - // flag indicating whether the test for given ase should pass. - shouldPass bool - // expected Result. - expectedData []byte - err error - }{ - // Test 1 - chmod 000 bucket/test-object1 - {bucketName, "test-object1", "test-object1", 0, int64(len(bytesData[0].byteData)), buffers[0], buffers[0], false, bytesData[0].byteData, PrefixAccessDenied{Bucket: bucketName, Object: "test-object1"}}, - // Test 2 - chmod 000 bucket/dir/ - {bucketName, "dir/test-object2", "dir", 0, int64(len(bytesData[0].byteData)), buffers[0], buffers[0], false, bytesData[0].byteData, PrefixAccessDenied{Bucket: bucketName, Object: "dir/test-object2"}}, - // Test 3 - chmod 000 bucket/ - {bucketName, "test-object3", "", 0, int64(len(bytesData[0].byteData)), buffers[0], buffers[0], false, bytesData[0].byteData, PrefixAccessDenied{Bucket: bucketName, Object: "test-object3"}}, - } - - for i, testCase := range testCases { - for _, d := range disks { - err = os.Chmod(d+SlashSeparator+testCase.bucketName+SlashSeparator+testCase.chmodPath, 0) - if err != nil { - t.Fatalf("Test %d, Unable to chmod: %v", i+1, err) - } - } - - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{}) - if err != nil && testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) - } - if err == nil && !testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead.", i+1, instanceType, testCase.err.Error()) - } - // Failed as expected, but does it fail for the expected reason. - if err != nil && !testCase.shouldPass { - if !strings.Contains(err.Error(), testCase.err.Error()) { - t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.err.Error(), err.Error()) - } - } - // Since there are cases for which GetObject fails, this is - // necessary. Test passes as expected, but the output values - // are verified for correctness here. - if err == nil && testCase.shouldPass { - if !bytes.Equal(testCase.expectedData, testCase.getObjectData.Bytes()) { - t.Errorf("Test %d: %s: Data Mismatch: Expected data and the fetched data from GetObject doesn't match.", i+1, instanceType) - } - // empty the buffer so that it can be used to further cases. - testCase.getObjectData.Reset() - } - } - -} - -// Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup. -func TestGetObjectDiskNotFound(t *testing.T) { - ExecObjectLayerDiskAlteredTest(t, testGetObjectDiskNotFound) -} - -// ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated. -// Before the Get Object call Erasure disks are moved so that the quorum just holds. -func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) { - // Setup for the tests. - bucketName := getRandomBucketName() - objectName := "test-object" - // create bucket. - err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{}) - // Stop the test if creation of the bucket fails. - if err != nil { - t.Fatalf("%s : %s", instanceType, err.Error()) - } - - // set of byte data for PutObject. - // object has to be created before running tests for GetObject. - // this is required even to assert the GetObject data, - // since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical. - bytesData := []struct { - byteData []byte - }{ - {generateBytesData(6 * humanize.MiByte)}, - } - // set of inputs for uploading the objects before tests for downloading is done. - putObjectInputs := []struct { - bucketName string - objectName string - contentLength int64 - textData []byte - metaData map[string]string - }{ - // case - 1. - {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, - } - // iterate through the above set of inputs and upkoad the object. - for i, input := range putObjectInputs { - // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), ObjectOptions{UserDefined: input.metaData}) - // if object upload fails stop the test. - if err != nil { - t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) - } - } - - // Take 4 disks down before GetObject is called, one more we loose quorum on 16 disk node. - for _, disk := range disks[:4] { - os.RemoveAll(disk) - } - - // set of empty buffers used to fill GetObject data. - buffers := []*bytes.Buffer{ - new(bytes.Buffer), - new(bytes.Buffer), - } - - // test cases with set of inputs - testCases := []struct { - bucketName string - objectName string - startOffset int64 - length int64 - // data obtained/fetched from GetObject. - getObjectData *bytes.Buffer - // writer which governs the write into the `getObjectData`. - writer io.Writer - // flag indicating whether the test for given ase should pass. - shouldPass bool - // expected Result. - expectedData []byte - err error - }{ - // Test case 1-4. - // Cases with invalid bucket names. - {".test", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: .test")}, - {"------", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: ------")}, - {"$this-is-not-valid-too", "obj", 0, 0, nil, nil, false, - []byte(""), fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")}, - {"a", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: a")}, - // Test case - 5. - // Case with invalid object names. - {bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"/")}, - // Test case - 7. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF}, - // Test case with start offset set to 0 and length set to size of the object. - // Fetching the entire object. - // Test case - 8. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], true, bytesData[0].byteData, nil}, - // Test case with `length` parameter set to a negative value. - // Test case - 9. - {bucketName, objectName, 0, int64(-1), buffers[1], buffers[1], true, bytesData[0].byteData, nil}, - // Test case with `length` parameter set to a negative value and offset is positive. - // Test case - 10. - {bucketName, objectName, 1, int64(-1), buffers[1], buffers[1], true, bytesData[0].byteData[1:], nil}, - // Test case with content-range 1 to objectSize . - // Test case - 11. - {bucketName, objectName, 1, int64(len(bytesData[0].byteData) - 1), buffers[1], buffers[1], true, bytesData[0].byteData[1:], nil}, - // Test case with content-range 100 to objectSize - 100. - // Test case - 12. - {bucketName, objectName, 100, int64(len(bytesData[0].byteData) - 200), buffers[1], buffers[1], true, - bytesData[0].byteData[100 : len(bytesData[0].byteData)-100], nil}, - // Test case with offset greater than the size of the object - // Test case - 13. - {bucketName, objectName, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), buffers[0], - NewEOFWriter(buffers[0], 100), false, []byte{}, - InvalidRange{int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}}, - // Test case with offset greater than the size of the object. - // Test case - 14. - {bucketName, objectName, -1, int64(len(bytesData[0].byteData)), buffers[0], new(bytes.Buffer), false, []byte{}, errUnexpected}, - // Test case length parameter is more than the object size. - // Test case - 15. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData) + 1), buffers[1], buffers[1], false, bytesData[0].byteData, - InvalidRange{0, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData))}}, - // Test case with offset + length > objectSize parameter set to a negative value. - // Test case - 16. - {bucketName, objectName, 2, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], false, bytesData[0].byteData, - InvalidRange{2, int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}}, - // Test case with the writer set to nil. - // Test case - 17. - {bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], nil, false, bytesData[0].byteData, errUnexpected}, - } - - for i, testCase := range testCases { - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{}) - if err != nil && testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) - } - if err == nil && !testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead.", i+1, instanceType, testCase.err.Error()) - } - // Failed as expected, but does it fail for the expected reason. - if err != nil && !testCase.shouldPass { - if !strings.Contains(err.Error(), testCase.err.Error()) { - t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.err.Error(), err.Error()) - } - } - // Since there are cases for which GetObject fails, this is - // necessary. Test passes as expected, but the output values - // are verified for correctness here. - if err == nil && testCase.shouldPass { - if !bytes.Equal(testCase.expectedData, testCase.getObjectData.Bytes()) { - t.Errorf("Test %d: %s: Data Mismatch: Expected data and the fetched data from GetObject doesn't match.", i+1, instanceType) - } - // empty the buffer so that it can be used to further cases. - testCase.getObjectData.Reset() - } - } -} - -// Benchmarks for ObjectLayer.GetObject(). -// The intent is to benchmark GetObject for various sizes ranging from few bytes to 100MB. -// Also each of these Benchmarks are run both Erasure and FS backends. - -// BenchmarkGetObjectVerySmallFS - Benchmark FS.GetObject() for object size of 10 bytes. -func BenchmarkGetObjectVerySmallFS(b *testing.B) { - benchmarkGetObject(b, "FS", 10) -} - -// BenchmarkGetObjectVerySmallErasure - Benchmark Erasure.GetObject() for object size of 10 bytes. -func BenchmarkGetObjectVerySmallErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 10) -} - -// BenchmarkGetObject10KbFS - Benchmark FS.GetObject() for object size of 10KB. -func BenchmarkGetObject10KbFS(b *testing.B) { - benchmarkGetObject(b, "FS", 10*humanize.KiByte) -} - -// BenchmarkGetObject10KbErasure - Benchmark Erasure.GetObject() for object size of 10KB. -func BenchmarkGetObject10KbErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 10*humanize.KiByte) -} - -// BenchmarkGetObject100KbFS - Benchmark FS.GetObject() for object size of 100KB. -func BenchmarkGetObject100KbFS(b *testing.B) { - benchmarkGetObject(b, "FS", 100*humanize.KiByte) -} - -// BenchmarkGetObject100KbErasure - Benchmark Erasure.GetObject() for object size of 100KB. -func BenchmarkGetObject100KbErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 100*humanize.KiByte) -} - -// BenchmarkGetObject1MbFS - Benchmark FS.GetObject() for object size of 1MB. -func BenchmarkGetObject1MbFS(b *testing.B) { - benchmarkGetObject(b, "FS", 1*humanize.MiByte) -} - -// BenchmarkGetObject1MbErasure - Benchmark Erasure.GetObject() for object size of 1MB. -func BenchmarkGetObject1MbErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 1*humanize.MiByte) -} - -// BenchmarkGetObject5MbFS - Benchmark FS.GetObject() for object size of 5MB. -func BenchmarkGetObject5MbFS(b *testing.B) { - benchmarkGetObject(b, "FS", 5*humanize.MiByte) -} - -// BenchmarkGetObject5MbErasure - Benchmark Erasure.GetObject() for object size of 5MB. -func BenchmarkGetObject5MbErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 5*humanize.MiByte) -} - -// BenchmarkGetObject10MbFS - Benchmark FS.GetObject() for object size of 10MB. -func BenchmarkGetObject10MbFS(b *testing.B) { - benchmarkGetObject(b, "FS", 10*humanize.MiByte) -} - -// BenchmarkGetObject10MbErasure - Benchmark Erasure.GetObject() for object size of 10MB. -func BenchmarkGetObject10MbErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 10*humanize.MiByte) -} - -// BenchmarkGetObject25MbFS - Benchmark FS.GetObject() for object size of 25MB. -func BenchmarkGetObject25MbFS(b *testing.B) { - benchmarkGetObject(b, "FS", 25*humanize.MiByte) - -} - -// BenchmarkGetObject25MbErasure - Benchmark Erasure.GetObject() for object size of 25MB. -func BenchmarkGetObject25MbErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 25*humanize.MiByte) -} - -// BenchmarkGetObject50MbFS - Benchmark FS.GetObject() for object size of 50MB. -func BenchmarkGetObject50MbFS(b *testing.B) { - benchmarkGetObject(b, "FS", 50*humanize.MiByte) -} - -// BenchmarkGetObject50MbErasure - Benchmark Erasure.GetObject() for object size of 50MB. -func BenchmarkGetObject50MbErasure(b *testing.B) { - benchmarkGetObject(b, "Erasure", 50*humanize.MiByte) -} - -// parallel benchmarks for ObjectLayer.GetObject() . - -// BenchmarkGetObjectParallelVerySmallFS - Benchmark FS.GetObject() for object size of 10 bytes. -func BenchmarkGetObjectParallelVerySmallFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 10) -} - -// BenchmarkGetObjectParallelVerySmallErasure - Benchmark Erasure.GetObject() for object size of 10 bytes. -func BenchmarkGetObjectParallelVerySmallErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 10) -} - -// BenchmarkGetObjectParallel10KbFS - Benchmark FS.GetObject() for object size of 10KB. -func BenchmarkGetObjectParallel10KbFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 10*humanize.KiByte) -} - -// BenchmarkGetObjectParallel10KbErasure - Benchmark Erasure.GetObject() for object size of 10KB. -func BenchmarkGetObjectParallel10KbErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 10*humanize.KiByte) -} - -// BenchmarkGetObjectParallel100KbFS - Benchmark FS.GetObject() for object size of 100KB. -func BenchmarkGetObjectParallel100KbFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 100*humanize.KiByte) -} - -// BenchmarkGetObjectParallel100KbErasure - Benchmark Erasure.GetObject() for object size of 100KB. -func BenchmarkGetObjectParallel100KbErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 100*humanize.KiByte) -} - -// BenchmarkGetObjectParallel1MbFS - Benchmark FS.GetObject() for object size of 1MB. -func BenchmarkGetObjectParallel1MbFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 1*humanize.MiByte) -} - -// BenchmarkGetObjectParallel1MbErasure - Benchmark Erasure.GetObject() for object size of 1MB. -func BenchmarkGetObjectParallel1MbErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 1*humanize.MiByte) -} - -// BenchmarkGetObjectParallel5MbFS - Benchmark FS.GetObject() for object size of 5MB. -func BenchmarkGetObjectParallel5MbFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 5*humanize.MiByte) -} - -// BenchmarkGetObjectParallel5MbErasure - Benchmark Erasure.GetObject() for object size of 5MB. -func BenchmarkGetObjectParallel5MbErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 5*humanize.MiByte) -} - -// BenchmarkGetObjectParallel10MbFS - Benchmark FS.GetObject() for object size of 10MB. -func BenchmarkGetObjectParallel10MbFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 10*humanize.MiByte) -} - -// BenchmarkGetObjectParallel10MbErasure - Benchmark Erasure.GetObject() for object size of 10MB. -func BenchmarkGetObjectParallel10MbErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 10*humanize.MiByte) -} - -// BenchmarkGetObjectParallel25MbFS - Benchmark FS.GetObject() for object size of 25MB. -func BenchmarkGetObjectParallel25MbFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 25*humanize.MiByte) - -} - -// BenchmarkGetObjectParallel25MbErasure - Benchmark Erasure.GetObject() for object size of 25MB. -func BenchmarkGetObjectParallel25MbErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 25*humanize.MiByte) -} - -// BenchmarkGetObjectParallel50MbFS - Benchmark FS.GetObject() for object size of 50MB. -func BenchmarkGetObjectParallel50MbFS(b *testing.B) { - benchmarkGetObjectParallel(b, "FS", 50*humanize.MiByte) -} - -// BenchmarkGetObjectParallel50MbErasure - Benchmark Erasure.GetObject() for object size of 50MB. -func BenchmarkGetObjectParallel50MbErasure(b *testing.B) { - benchmarkGetObjectParallel(b, "Erasure", 50*humanize.MiByte) -} diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index d6ecd0ef6..c78e03c8f 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -112,7 +112,6 @@ type ObjectLayer interface { // IMPORTANTLY, when implementations return err != nil, this // function MUST NOT return a non-nil ReadCloser. GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error) - GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) @@ -162,3 +161,23 @@ type ObjectLayer interface { GetObjectTags(context.Context, string, string, ObjectOptions) (*tags.Tags, error) DeleteObjectTags(context.Context, string, string, ObjectOptions) (ObjectInfo, error) } + +// GetObject - TODO(aead): This function just acts as an adapter for GetObject tests and benchmarks +// since the GetObject method of the ObjectLayer interface has been removed. Once, the +// tests are adjusted to use GetObjectNInfo this function can be removed. +func GetObject(ctx context.Context, api ObjectLayer, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) { + var header http.Header + if etag != "" { + header.Set("ETag", etag) + } + Range := &HTTPRangeSpec{Start: startOffset, End: startOffset + length} + + reader, err := api.GetObjectNInfo(ctx, bucket, object, Range, header, readLock, opts) + if err != nil { + return err + } + defer reader.Close() + + _, err = io.Copy(writer, reader) + return err +} diff --git a/cmd/object_api_suite_test.go b/cmd/object_api_suite_test.go index 9aee8d74e..21cefebbe 100644 --- a/cmd/object_api_suite_test.go +++ b/cmd/object_api_suite_test.go @@ -210,7 +210,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH for key, value := range objects { var byteBuffer bytes.Buffer - err = obj.GetObject(context.Background(), "bucket", key, 0, int64(len(value)), &byteBuffer, "", opts) + err = GetObject(context.Background(), obj, "bucket", key, 0, int64(len(value)), &byteBuffer, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -461,7 +461,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan } var bytesBuffer bytes.Buffer - err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer, "", opts) + err = GetObject(context.Background(), obj, "bucket", "object", 0, length, &bytesBuffer, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -614,7 +614,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) { if err != nil { t.Fatalf("%s: %s", instanceType, err) } - err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer1, "", opts) + err = GetObject(context.Background(), obj, "bucket", "object", 0, length, &bytesBuffer1, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -627,7 +627,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) { if err != nil { t.Fatalf("%s: %s", instanceType, err) } - err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer2, "", opts) + err = GetObject(context.Background(), obj, "bucket", "object", 0, length, &bytesBuffer2, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -658,7 +658,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle } var bytesBuffer bytes.Buffer - err = obj.GetObject(context.Background(), "bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "", opts) + err = GetObject(context.Background(), obj, "bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } diff --git a/cmd/web-handlers_test.go b/cmd/web-handlers_test.go index 292686aeb..6c49f26f1 100644 --- a/cmd/web-handlers_test.go +++ b/cmd/web-handlers_test.go @@ -789,7 +789,7 @@ func testUploadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandler } var byteBuffer bytes.Buffer - err = obj.GetObject(context.Background(), bucketName, objectName, 0, int64(len(content)), &byteBuffer, "", ObjectOptions{}) + err = GetObject(context.Background(), obj, bucketName, objectName, 0, int64(len(content)), &byteBuffer, "", ObjectOptions{}) if err != nil { t.Fatalf("Failed, %v", err) }